@qvac/inference 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (569) hide show
  1. package/NOTICE +111 -60
  2. package/README.md +4 -2
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +136 -45
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +2 -1
  13. package/dist/api/completion-stream.js +4 -1
  14. package/dist/api/decide.js +1 -1
  15. package/dist/api/delete-cache.d.ts +7 -1
  16. package/dist/api/delete-cache.js +11 -1
  17. package/dist/api/diffusion.d.ts +32 -2
  18. package/dist/api/diffusion.js +32 -2
  19. package/dist/api/download-asset.js +1 -1
  20. package/dist/api/embed.d.ts +82 -29
  21. package/dist/api/embed.js +64 -18
  22. package/dist/api/finetune.js +1 -1
  23. package/dist/api/get-model-info.d.ts +1 -1
  24. package/dist/api/index.d.ts +40 -38
  25. package/dist/api/index.js +39 -37
  26. package/dist/api/load-model.js +2 -2
  27. package/dist/api/logging-stream-registry.js +11 -7
  28. package/dist/api/logging-stream.d.ts +11 -1
  29. package/dist/api/logging-stream.js +11 -2
  30. package/dist/api/rag.d.ts +1 -1
  31. package/dist/api/rag.js +1 -1
  32. package/dist/api/registry.d.ts +18 -7
  33. package/dist/api/registry.js +37 -13
  34. package/dist/api/rerank.d.ts +17 -0
  35. package/dist/api/rerank.js +36 -0
  36. package/dist/api/subscribe-logs.js +5 -3
  37. package/dist/api/text-to-speech.d.ts +14 -5
  38. package/dist/api/text-to-speech.js +83 -29
  39. package/dist/api/transcribe.d.ts +6 -2
  40. package/dist/api/transcribe.js +7 -1
  41. package/dist/api/unload-model.js +1 -1
  42. package/dist/api/vector-index.d.ts +25 -3
  43. package/dist/api/vector-index.js +17 -2
  44. package/dist/api/video-request.d.ts +15 -15
  45. package/dist/api/video.js +1 -1
  46. package/dist/api/vla.js +1 -1
  47. package/dist/api/world-result.js +1 -1
  48. package/dist/api/world.d.ts +1 -1
  49. package/dist/api/world.js +1 -1
  50. package/dist/config/resolve-config.d.ts +1 -1
  51. package/dist/config/resolve-config.js +1 -1
  52. package/dist/constants/index.d.ts +2 -2
  53. package/dist/constants/index.js +2 -2
  54. package/dist/dispatch.d.ts +11 -4
  55. package/dist/dispatch.js +80 -18
  56. package/dist/engine.d.ts +1 -1
  57. package/dist/errors/index.d.ts +76 -0
  58. package/dist/errors/index.js +141 -1
  59. package/dist/handlers/assess-model-fit.d.ts +3 -21
  60. package/dist/handlers/assess-model-fit.js +4 -303
  61. package/dist/handlers/cancelHandler.d.ts +10 -9
  62. package/dist/handlers/cancelHandler.js +18 -14
  63. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  64. package/dist/handlers/completion-orchestrate.js +166 -24
  65. package/dist/handlers/delete-cache.d.ts +2 -1
  66. package/dist/handlers/delete-cache.js +6 -1
  67. package/dist/handlers/download-asset.js +13 -6
  68. package/dist/handlers/finetune.js +1 -1
  69. package/dist/handlers/get-loaded-model-info.js +3 -3
  70. package/dist/handlers/get-model-info.js +72 -3
  71. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
  72. package/dist/handlers/load-model/coreml-sidecars.js +51 -91
  73. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  74. package/dist/handlers/load-model/download-lock.js +213 -0
  75. package/dist/handlers/load-model/download-manager.d.ts +11 -3
  76. package/dist/handlers/load-model/download-manager.js +62 -11
  77. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  78. package/dist/handlers/load-model/download-stats.js +3 -3
  79. package/dist/handlers/load-model/handler.js +97 -41
  80. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  81. package/dist/handlers/load-model/http.d.ts +1 -1
  82. package/dist/handlers/load-model/http.js +330 -208
  83. package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
  84. package/dist/handlers/load-model/hyperdrive.js +125 -52
  85. package/dist/handlers/load-model/index.d.ts +2 -2
  86. package/dist/handlers/load-model/index.js +2 -2
  87. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  88. package/dist/handlers/load-model/registry-client-options.js +4 -2
  89. package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
  90. package/dist/handlers/load-model/registry-companion-set.js +20 -2
  91. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  92. package/dist/handlers/load-model/registry-delete.js +269 -0
  93. package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
  94. package/dist/handlers/load-model/registry-download-utils.js +95 -24
  95. package/dist/handlers/load-model/registry.d.ts +22 -2
  96. package/dist/handlers/load-model/registry.js +312 -179
  97. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  98. package/dist/handlers/load-model/resolve-session.js +11 -3
  99. package/dist/handlers/load-model/resolve.d.ts +1 -1
  100. package/dist/handlers/load-model/resolve.js +9 -6
  101. package/dist/handlers/load-model/types.d.ts +2 -0
  102. package/dist/handlers/logging-stream.d.ts +2 -1
  103. package/dist/handlers/logging-stream.js +35 -7
  104. package/dist/handlers/plugin-invoke.js +152 -39
  105. package/dist/handlers/registry.d.ts +2 -2
  106. package/dist/handlers/registry.js +108 -29
  107. package/dist/handlers/rpc-server.js +7 -7
  108. package/dist/handlers/session.d.ts +11 -0
  109. package/dist/handlers/session.js +40 -0
  110. package/dist/handlers/types.d.ts +7 -1
  111. package/dist/handlers/unload-model.js +8 -3
  112. package/dist/handlers/vector-index.js +118 -19
  113. package/dist/index.d.ts +1 -1
  114. package/dist/index.js +1 -1
  115. package/dist/logging/addon.d.ts +1 -1
  116. package/dist/logging/addon.js +1 -1
  117. package/dist/logging/index.d.ts +5 -5
  118. package/dist/logging/index.js +4 -4
  119. package/dist/logging/logger.d.ts +1 -1
  120. package/dist/logging/logger.js +3 -3
  121. package/dist/logging/utils.js +11 -3
  122. package/dist/model-cache-layout.d.ts +1 -0
  123. package/dist/model-cache-layout.js +2 -0
  124. package/dist/model-fit/assess.d.ts +23 -0
  125. package/dist/model-fit/assess.js +110 -0
  126. package/dist/model-fit/describe.d.ts +12 -0
  127. package/dist/model-fit/describe.js +29 -0
  128. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
  129. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
  130. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
  131. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
  132. package/dist/model-fit/fit-load.d.ts +23 -0
  133. package/dist/model-fit/fit-load.js +29 -0
  134. package/dist/model-fit/load-check.d.ts +34 -0
  135. package/dist/model-fit/load-check.js +77 -0
  136. package/dist/model-fit/run/engine.d.ts +12 -0
  137. package/dist/model-fit/run/engine.js +9 -0
  138. package/dist/model-fit/run/run.d.ts +46 -0
  139. package/dist/model-fit/run/run.js +208 -0
  140. package/dist/model-fit/sum.d.ts +32 -0
  141. package/dist/model-fit/sum.js +60 -0
  142. package/dist/models/registry/index.d.ts +1 -1
  143. package/dist/models/registry/index.js +2 -2
  144. package/dist/models/registry/models.d.ts +1377 -130
  145. package/dist/models/registry/models.js +3622 -2312
  146. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  147. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  148. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  149. package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
  150. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  151. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  152. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  153. package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  154. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  155. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
  156. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  157. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  158. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  159. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  160. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  161. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
  162. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  163. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
  164. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
  165. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
  166. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  167. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  168. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
  169. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
  170. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
  171. package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
  172. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  173. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  174. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  175. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  176. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  177. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  178. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  179. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
  180. package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
  181. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  182. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  183. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
  184. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  185. package/dist/plugins/builtin/index.d.ts +16 -14
  186. package/dist/plugins/builtin/index.js +16 -14
  187. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  188. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  189. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
  190. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  191. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  192. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  193. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
  194. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  195. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
  196. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
  197. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  198. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  199. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
  200. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
  201. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
  202. package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
  203. package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
  204. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
  205. package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
  206. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  207. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
  208. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
  209. package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
  210. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  211. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  212. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
  213. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
  214. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  215. package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
  216. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  217. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  218. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
  219. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
  220. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
  221. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
  222. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
  223. package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
  224. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  225. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  226. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
  227. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
  228. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
  229. package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
  230. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  231. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  232. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  233. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  234. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
  235. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
  236. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  237. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  238. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
  239. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
  240. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
  241. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
  242. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
  243. package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
  244. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  245. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  246. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  247. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  248. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  249. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  250. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  251. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  252. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  253. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
  254. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
  255. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  256. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
  257. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  258. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  259. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
  260. package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
  261. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
  262. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
  263. package/dist/plugins/index.d.ts +1 -1
  264. package/dist/plugins/index.js +2 -2
  265. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  266. package/dist/plugins/ops/bci-transcribe.js +18 -4
  267. package/dist/plugins/ops/cancel.d.ts +1 -0
  268. package/dist/plugins/ops/cancel.js +11 -2
  269. package/dist/plugins/ops/embed.d.ts +5 -3
  270. package/dist/plugins/ops/embed.js +89 -34
  271. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
  272. package/dist/plugins/ops/kv-cache-utils.js +21 -10
  273. package/dist/plugins/ops/load-model.d.ts +10 -2
  274. package/dist/plugins/ops/load-model.js +45 -19
  275. package/dist/plugins/ops/transcribe.d.ts +5 -4
  276. package/dist/plugins/ops/transcribe.js +63 -28
  277. package/dist/plugins/ops/translate.d.ts +2 -1
  278. package/dist/plugins/ops/translate.js +7 -4
  279. package/dist/plugins/ops/unload-model.d.ts +17 -1
  280. package/dist/plugins/ops/unload-model.js +32 -3
  281. package/dist/plugins/turbovec-backend.d.ts +1 -1
  282. package/dist/plugins/turbovec-backend.js +21 -6
  283. package/dist/profiling/aggregator.d.ts +1 -1
  284. package/dist/profiling/aggregator.js +1 -1
  285. package/dist/profiling/controller.d.ts +1 -1
  286. package/dist/profiling/controller.js +2 -2
  287. package/dist/profiling/events.d.ts +1 -1
  288. package/dist/profiling/events.js +2 -2
  289. package/dist/profiling/exporters.d.ts +1 -1
  290. package/dist/profiling/exporters.js +3 -3
  291. package/dist/profiling/index.d.ts +13 -13
  292. package/dist/profiling/index.js +12 -12
  293. package/dist/profiling/operation-metrics.d.ts +1 -1
  294. package/dist/profiling/operation-metrics.js +24 -21
  295. package/dist/profiling/operation-wrappers.js +3 -3
  296. package/dist/profiling/profiler.js +1 -1
  297. package/dist/rag/embedding-scope.d.ts +16 -0
  298. package/dist/rag/embedding-scope.js +81 -0
  299. package/dist/rag/handler.d.ts +25 -0
  300. package/dist/rag/handler.js +51 -17
  301. package/dist/rag/handlers/close-workspace.js +1 -1
  302. package/dist/rag/handlers/delete-embeddings.js +1 -1
  303. package/dist/rag/handlers/delete-workspace.js +1 -1
  304. package/dist/rag/handlers/ingest.js +3 -7
  305. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  306. package/dist/rag/handlers/list-workspaces.js +1 -1
  307. package/dist/rag/handlers/reindex.js +1 -1
  308. package/dist/rag/handlers/save-embeddings.js +1 -1
  309. package/dist/rag/handlers/search.d.ts +10 -1
  310. package/dist/rag/handlers/search.js +17 -8
  311. package/dist/rag/index.d.ts +11 -11
  312. package/dist/rag/index.js +11 -11
  313. package/dist/rag/rag-operation-manager.d.ts +27 -3
  314. package/dist/rag/rag-operation-manager.js +10 -27
  315. package/dist/rag/rag-workspace-manager.js +1 -1
  316. package/dist/registry.js +7 -1
  317. package/dist/resources/collector.d.ts +71 -37
  318. package/dist/resources/collector.js +9 -2
  319. package/dist/resources/instance.d.ts +1 -1
  320. package/dist/resources/instance.js +1 -1
  321. package/dist/resources/native.d.ts +1 -1
  322. package/dist/resources/normalize.d.ts +1 -1
  323. package/dist/resources/types.d.ts +2 -0
  324. package/dist/rpc/inventory.js +1 -12
  325. package/dist/runtime/env.d.ts +5 -1
  326. package/dist/runtime/env.js +6 -21
  327. package/dist/runtime/index.d.ts +8 -5
  328. package/dist/runtime/index.js +4 -2
  329. package/dist/runtime/lifecycle.d.ts +6 -4
  330. package/dist/runtime/lifecycle.js +16 -12
  331. package/dist/runtime/model-config-utils.js +8 -1
  332. package/dist/runtime/model-registry.d.ts +30 -4
  333. package/dist/runtime/model-registry.js +48 -2
  334. package/dist/runtime/model-work-lock.d.ts +13 -0
  335. package/dist/runtime/model-work-lock.js +48 -0
  336. package/dist/runtime/registry-client.d.ts +47 -1
  337. package/dist/runtime/registry-client.js +446 -88
  338. package/dist/runtime/request-context.d.ts +11 -3
  339. package/dist/runtime/request-context.js +39 -2
  340. package/dist/runtime/request-registry.d.ts +56 -8
  341. package/dist/runtime/request-registry.js +190 -72
  342. package/dist/runtime/runtime-lifecycle.js +10 -1
  343. package/dist/runtime/session-id.d.ts +18 -0
  344. package/dist/runtime/session-id.js +20 -0
  345. package/dist/runtime/session-registry.d.ts +39 -0
  346. package/dist/runtime/session-registry.js +114 -0
  347. package/dist/runtime/state.d.ts +1 -1
  348. package/dist/runtime/state.js +2 -2
  349. package/dist/runtime/vector-index-backend.d.ts +6 -2
  350. package/dist/runtime/vector-index-registry.d.ts +26 -5
  351. package/dist/runtime/vector-index-registry.js +88 -27
  352. package/dist/schemas/abort-signal.d.ts +8 -0
  353. package/dist/schemas/assess-model-fit.d.ts +138 -282
  354. package/dist/schemas/assess-model-fit.js +85 -219
  355. package/dist/schemas/audio-gen.d.ts +291 -25
  356. package/dist/schemas/audio-gen.js +130 -26
  357. package/dist/schemas/batch-completion-stream.d.ts +113 -15
  358. package/dist/schemas/batch-completion-stream.js +10 -6
  359. package/dist/schemas/bci-config.d.ts +3 -1
  360. package/dist/schemas/bci-config.js +2 -2
  361. package/dist/schemas/bci.d.ts +55 -4
  362. package/dist/schemas/bci.js +13 -18
  363. package/dist/schemas/calc-tokens.d.ts +141 -0
  364. package/dist/schemas/calc-tokens.js +53 -0
  365. package/dist/schemas/cancel.d.ts +31 -0
  366. package/dist/schemas/cancel.js +11 -2
  367. package/dist/schemas/classification.d.ts +3 -0
  368. package/dist/schemas/classification.js +6 -2
  369. package/dist/schemas/common.d.ts +2209 -348
  370. package/dist/schemas/common.js +52 -33
  371. package/dist/schemas/completion-event.d.ts +11 -2
  372. package/dist/schemas/completion-event.js +5 -2
  373. package/dist/schemas/completion-stream.d.ts +173 -19
  374. package/dist/schemas/completion-stream.js +117 -20
  375. package/dist/schemas/config.d.ts +2064 -178
  376. package/dist/schemas/config.js +21 -23
  377. package/dist/schemas/decision.d.ts +12 -2
  378. package/dist/schemas/decision.js +16 -1
  379. package/dist/schemas/download-asset.d.ts +13 -6
  380. package/dist/schemas/download-asset.js +4 -7
  381. package/dist/schemas/embed.d.ts +286 -3
  382. package/dist/schemas/embed.js +162 -16
  383. package/dist/schemas/engine-addon-map.d.ts +3 -1
  384. package/dist/schemas/engine-addon-map.js +5 -3
  385. package/dist/schemas/errors.d.ts +13 -0
  386. package/dist/schemas/errors.js +71 -1
  387. package/dist/schemas/finetune.d.ts +14 -4
  388. package/dist/schemas/finetune.js +5 -6
  389. package/dist/schemas/heartbeat.d.ts +1 -0
  390. package/dist/schemas/heartbeat.js +5 -2
  391. package/dist/schemas/image-embedding.d.ts +27 -0
  392. package/dist/schemas/image-embedding.js +107 -0
  393. package/dist/schemas/index.d.ts +55 -50
  394. package/dist/schemas/index.js +54 -49
  395. package/dist/schemas/lifecycle.d.ts +3 -0
  396. package/dist/schemas/lifecycle.js +13 -6
  397. package/dist/schemas/llamacpp-config.d.ts +443 -20
  398. package/dist/schemas/llamacpp-config.js +188 -14
  399. package/dist/schemas/load-model.d.ts +8940 -1952
  400. package/dist/schemas/load-model.js +73 -9
  401. package/dist/schemas/logging-stream.d.ts +1 -0
  402. package/dist/schemas/logging-stream.js +5 -2
  403. package/dist/schemas/mcp-adapter.d.ts +1 -1
  404. package/dist/schemas/model-config-schemas.d.ts +731 -111
  405. package/dist/schemas/model-config-schemas.js +16 -12
  406. package/dist/schemas/model-fit-policy.d.ts +2 -2
  407. package/dist/schemas/model-info.d.ts +54 -56
  408. package/dist/schemas/model-info.js +18 -7
  409. package/dist/schemas/model-ops.d.ts +12 -0
  410. package/dist/schemas/model-ops.js +33 -5
  411. package/dist/schemas/model-src-utils.d.ts +14 -6
  412. package/dist/schemas/model-src-utils.js +2 -2
  413. package/dist/schemas/model-types.d.ts +12 -4
  414. package/dist/schemas/model-types.js +2 -0
  415. package/dist/schemas/ocr.d.ts +11 -4
  416. package/dist/schemas/ocr.js +7 -3
  417. package/dist/schemas/plugin.d.ts +70 -8
  418. package/dist/schemas/plugin.js +32 -4
  419. package/dist/schemas/profiling.d.ts +32 -32
  420. package/dist/schemas/profiling.js +1 -1
  421. package/dist/schemas/rag.d.ts +9 -0
  422. package/dist/schemas/rag.js +11 -17
  423. package/dist/schemas/registry.d.ts +38 -10
  424. package/dist/schemas/registry.js +16 -7
  425. package/dist/schemas/reload-config.d.ts +45 -6
  426. package/dist/schemas/reload-config.js +13 -6
  427. package/dist/schemas/request-base.d.ts +39 -0
  428. package/dist/schemas/request-base.js +64 -0
  429. package/dist/schemas/rerank.d.ts +132 -0
  430. package/dist/schemas/rerank.js +104 -0
  431. package/dist/schemas/rpc-server.d.ts +3 -0
  432. package/dist/schemas/rpc-server.js +15 -15
  433. package/dist/schemas/sdcpp-config.d.ts +144 -18
  434. package/dist/schemas/sdcpp-config.js +123 -28
  435. package/dist/schemas/session.d.ts +37 -0
  436. package/dist/schemas/session.js +53 -0
  437. package/dist/schemas/system-resources.d.ts +364 -179
  438. package/dist/schemas/system-resources.js +8 -3
  439. package/dist/schemas/text-to-speech.d.ts +1408 -275
  440. package/dist/schemas/text-to-speech.js +547 -33
  441. package/dist/schemas/tools.d.ts +7 -0
  442. package/dist/schemas/tools.js +29 -0
  443. package/dist/schemas/transcription-config.d.ts +267 -2
  444. package/dist/schemas/transcription-config.js +138 -7
  445. package/dist/schemas/transcription.d.ts +94 -7
  446. package/dist/schemas/transcription.js +99 -22
  447. package/dist/schemas/translate.d.ts +4 -2
  448. package/dist/schemas/translate.js +6 -10
  449. package/dist/schemas/translation-config.d.ts +40 -20
  450. package/dist/schemas/translation-config.js +1 -1
  451. package/dist/schemas/tts-pcm.d.ts +35 -0
  452. package/dist/schemas/tts-pcm.js +69 -0
  453. package/dist/schemas/vector-index.d.ts +47 -1
  454. package/dist/schemas/vector-index.js +37 -10
  455. package/dist/schemas/vla.d.ts +3 -3
  456. package/dist/surface.d.ts +4 -3
  457. package/dist/surface.js +4 -3
  458. package/dist/utils/addon-responses.d.ts +71 -2
  459. package/dist/utils/aggregate-events.d.ts +1 -1
  460. package/dist/utils/aggregate-events.js +2 -2
  461. package/dist/utils/archive.d.ts +7 -0
  462. package/dist/utils/archive.js +89 -30
  463. package/dist/utils/asr-events.d.ts +5 -0
  464. package/dist/utils/asr-events.js +10 -1
  465. package/dist/utils/audio/decoder.js +3 -1
  466. package/dist/utils/audio-input.js +1 -1
  467. package/dist/utils/cache/auto-history.d.ts +2 -1
  468. package/dist/utils/cache/auto-history.js +5 -3
  469. package/dist/utils/cache/layout.d.ts +48 -0
  470. package/dist/utils/cache/layout.js +190 -0
  471. package/dist/utils/cache/paths.js +8 -10
  472. package/dist/utils/cache/types.d.ts +2 -1
  473. package/dist/utils/checksum.d.ts +7 -0
  474. package/dist/utils/checksum.js +59 -2
  475. package/dist/utils/completion-normalizer.js +287 -88
  476. package/dist/utils/gguf-architecture.d.ts +7 -0
  477. package/dist/utils/gguf-architecture.js +142 -0
  478. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  479. package/dist/utils/gguf-tensor-extractor.js +1 -50
  480. package/dist/utils/index.d.ts +13 -13
  481. package/dist/utils/index.js +13 -13
  482. package/dist/utils/mcp-adapter.d.ts +1 -1
  483. package/dist/utils/mcp-adapter.js +1 -1
  484. package/dist/utils/path-security.js +1 -1
  485. package/dist/utils/shard-utils.d.ts +0 -16
  486. package/dist/utils/shard-utils.js +14 -48
  487. package/dist/utils/tool-integration.d.ts +3 -2
  488. package/dist/utils/tool-integration.js +3 -6
  489. package/dist/utils/tools/defer.d.ts +50 -31
  490. package/dist/utils/tools/defer.js +115 -51
  491. package/dist/utils/tools/dialect.d.ts +15 -4
  492. package/dist/utils/tools/dialect.js +34 -5
  493. package/dist/utils/tools/index.d.ts +3 -3
  494. package/dist/utils/tools/index.js +3 -3
  495. package/dist/utils/tools/parser.js +8 -8
  496. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  497. package/dist/utils/tools/parsers/dsml.js +1 -1
  498. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  499. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  500. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  501. package/dist/utils/tools/parsers/harmony.js +1 -1
  502. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  503. package/dist/utils/tools/parsers/hermes.js +1 -1
  504. package/dist/utils/tools/parsers/json.d.ts +1 -1
  505. package/dist/utils/tools/parsers/json.js +1 -1
  506. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  507. package/dist/utils/tools/parsers/pythonic.js +1 -1
  508. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  509. package/dist/utils/tools/parsers/qwen35.js +1 -1
  510. package/dist/utils/transcribe-metadata.d.ts +28 -2
  511. package/dist/utils/transcribe-metadata.js +16 -0
  512. package/dist/utils/tts-stats.d.ts +43 -1
  513. package/dist/utils/tts-stats.js +120 -22
  514. package/dist/utils/until-aborted.d.ts +7 -0
  515. package/dist/utils/until-aborted.js +34 -0
  516. package/package.json +67 -55
  517. package/dist/models/registry/resource-profiles.d.ts +0 -17
  518. package/dist/models/registry/resource-profiles.js +0 -5657
  519. package/dist/resources/model-fit/assess.d.ts +0 -72
  520. package/dist/resources/model-fit/assess.js +0 -679
  521. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
  522. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
  523. package/dist/resources/model-fit/floor.d.ts +0 -31
  524. package/dist/resources/model-fit/floor.js +0 -65
  525. package/dist/resources/model-fit/kv-cache.d.ts +0 -23
  526. package/dist/resources/model-fit/kv-cache.js +0 -85
  527. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
  528. package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
  529. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  530. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
  531. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
  532. package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
  533. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
  534. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
  535. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
  536. package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
  537. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
  538. package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
  539. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
  540. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
  541. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
  542. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
  543. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
  544. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
  545. package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
  546. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
  547. package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
  548. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
  549. package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
  550. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  551. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  552. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  553. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  554. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  555. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  556. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  557. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
  558. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
  559. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  560. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  561. package/dist/resources/model-fit/platform.d.ts +0 -9
  562. package/dist/resources/model-fit/platform.js +0 -26
  563. package/dist/resources/model-fit/types.d.ts +0 -6
  564. package/dist/resources/model-fit/types.js +0 -1
  565. package/dist/schemas/model-resource-profile.d.ts +0 -108
  566. package/dist/schemas/model-resource-profile.js +0 -80
  567. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
  568. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
  569. /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
@@ -1,4 +1,4 @@
1
- import { parseToolCalls } from './tools/index.js';
1
+ import { parseToolCalls } from '../utils/tools/index.js';
2
2
  export const MAX_TOOL_FRAME_CHARS = 1_048_576;
3
3
  const DSML_INVOKE_FRAMES = [' ', '\n', '\t', '\r'].map((sep) => ({
4
4
  open: `<|DSML|invoke${sep}`,
@@ -74,14 +74,38 @@ export function createCompletionNormalizer(config) {
74
74
  // (could be the built-in `<think>` or a dialect-specific frame like
75
75
  // Harmony's `<|channel|>analysis<|message|>...<|end|>`).
76
76
  let activeThinkingFrame = null;
77
+ // Qwen-style templates open `<think>` in the prompt and the model sometimes
78
+ // skips `</think>` before calling a tool. A call parsed inside the block is
79
+ // held until what follows shows whether it was made or only quoted.
80
+ let toolFrameOpenedInThinking = false;
81
+ let heldThinkingCalls = null;
82
+ // Index into `thinkingParts` where the current built-in block starts.
83
+ let thinkingBlockStart = 0;
84
+ // Calls in the current block from the first one followed by more reasoning.
85
+ // That kind was quoted if the block closes and made if the turn ends with it
86
+ // open; calls made after it wait here so they are emitted in order.
87
+ let deferredCalls = [];
88
+ let undecidedAnswerStart = 0;
77
89
  // Content frame: inner streamed as it arrives; partial close tail buffered.
78
90
  let activeContentFrame = null;
91
+ // Set when a thinking frame closes before any content. Reasoning templates
92
+ // put a blank line between the close marker and the answer; it belongs to
93
+ // the framing, not the answer, so whitespace is dropped until the answer's
94
+ // first visible character.
95
+ let trimAfterThinking = false;
96
+ // The same framing whitespace sits inside a thinking frame (`<think>\n` ...
97
+ // `\n</think>`). Leading whitespace is dropped and trailing whitespace held
98
+ // back until more reasoning follows, so the close drops it. Both reset when a
99
+ // frame opens; a later frame starts on a new line so frames don't run together.
100
+ let trimThinkingStart = false;
101
+ let pendingThinkingWhitespace = '';
79
102
  // Trailing chars that could be a prefix of a watched marker, so markers
80
103
  // split across token pushes aren't missed by indexOf.
81
104
  let contentTail = '';
82
105
  let toolFramingTail = '';
83
106
  let thinkingFramingTail = '';
84
107
  let contentFramingTail = '';
108
+ let afterThinkingToolTail = '';
85
109
  const streamedToolCallKeys = new Set();
86
110
  const toolsEnabled = config.tools.length > 0 && config.capabilities.toolCalling !== 'none';
87
111
  const useToolFraming = toolsEnabled && config.capabilities.toolCalling === 'textParse';
@@ -110,6 +134,7 @@ export function createCompletionNormalizer(config) {
110
134
  }
111
135
  }
112
136
  registerFrames(toolFrames, 'tool');
137
+ const toolOpeners = toolFrames.map((f) => f.open);
113
138
  registerFrames(thinkingFrames, 'thinking');
114
139
  registerFrames(contentFrames, 'content');
115
140
  for (const t of dropTokens) {
@@ -126,6 +151,11 @@ export function createCompletionNormalizer(config) {
126
151
  for (const token of contentDropTokens) {
127
152
  normalized = normalized.split(token).join('');
128
153
  }
154
+ if (trimAfterThinking) {
155
+ normalized = normalized.trimStart();
156
+ if (normalized)
157
+ trimAfterThinking = false;
158
+ }
129
159
  if (!normalized)
130
160
  return;
131
161
  contentParts.push(normalized);
@@ -135,11 +165,25 @@ export function createCompletionNormalizer(config) {
135
165
  // is false (e.g. harmony's analysis channel, to strip markers from
136
166
  // `contentDelta`) — drop the inner here instead of buffering it.
137
167
  function emitThinking(events, text) {
138
- if (!text)
168
+ let normalized = text;
169
+ if (trimThinkingStart) {
170
+ normalized = normalized.trimStart();
171
+ if (!normalized)
172
+ return;
173
+ trimThinkingStart = false;
174
+ if (thinkingParts.length > 0)
175
+ normalized = '\n' + normalized;
176
+ }
177
+ const body = normalized.trimEnd();
178
+ if (!body) {
179
+ pendingThinkingWhitespace += normalized;
139
180
  return;
181
+ }
182
+ const out = pendingThinkingWhitespace + body;
183
+ pendingThinkingWhitespace = normalized.slice(body.length);
140
184
  if (config.captureThinking) {
141
- thinkingParts.push(text);
142
- events.push({ type: 'thinkingDelta', seq: nextSeq(), text });
185
+ thinkingParts.push(out);
186
+ events.push({ type: 'thinkingDelta', seq: nextSeq(), text: out });
143
187
  }
144
188
  }
145
189
  function getToolCallKey(call) {
@@ -149,8 +193,16 @@ export function createCompletionNormalizer(config) {
149
193
  streamedToolCallKeys.add(getToolCallKey(call));
150
194
  events.push({ type: 'toolCall', seq: nextSeq(), call });
151
195
  }
196
+ // A made call waits behind earlier undecided ones so the order is kept.
197
+ function emitMadeCall(events, call) {
198
+ if (deferredCalls.length > 0)
199
+ deferredCalls.push({ call, made: true });
200
+ else
201
+ emitStreamedToolCall(events, call);
202
+ }
152
203
  function emitFinalToolCall(events, call) {
153
- if (streamedToolCallKeys.has(getToolCallKey(call)))
204
+ const key = getToolCallKey(call);
205
+ if (streamedToolCallKeys.has(key))
154
206
  return;
155
207
  events.push({ type: 'toolCall', seq: nextSeq(), call });
156
208
  }
@@ -179,14 +231,37 @@ export function createCompletionNormalizer(config) {
179
231
  }
180
232
  return best;
181
233
  }
182
- function flushToolBuffer(events, closed) {
183
- const raw = toolCallBuffer;
184
- const frame = activeToolFrame;
234
+ function openToolFrame(marker, spec, fromThinking) {
235
+ toolCallBuffer = marker;
236
+ activeToolFrame = spec;
237
+ toolFrameOpenedInThinking = fromThinking;
238
+ state = 'toolFraming';
239
+ }
240
+ // Every exit from a tool frame goes through here, so no frame state outlives it.
241
+ function takeToolFrame() {
242
+ const taken = {
243
+ raw: toolCallBuffer + toolFramingTail,
244
+ frame: activeToolFrame,
245
+ openedInThinking: toolFrameOpenedInThinking
246
+ };
185
247
  toolCallBuffer = '';
186
- activeToolFrame = null;
187
248
  toolFramingTail = '';
249
+ activeToolFrame = null;
250
+ toolFrameOpenedInThinking = false;
251
+ return taken;
252
+ }
253
+ function flushToolBuffer(events, closed) {
254
+ const { raw, frame, openedInThinking } = takeToolFrame();
188
255
  if (!frame)
189
256
  return;
257
+ // The reasoning only mentioned the marker: still inside `<think>`.
258
+ function returnToThinking() {
259
+ trimThinkingStart = false;
260
+ emitThinking(events, raw);
261
+ activeThinkingFrame = BUILTIN_THINKING;
262
+ if (closed)
263
+ state = 'thinkingFraming';
264
+ }
190
265
  // Empty-payload guard. Strip frame markers and check the inner is
191
266
  // non-empty; surface a toolError on empty payloads so the literal
192
267
  // markers don't leak into contentDelta.
@@ -197,6 +272,8 @@ export function createCompletionNormalizer(config) {
197
272
  inner = inner.slice(0, inner.length - frame.close.length);
198
273
  }
199
274
  if (inner.trim().length === 0) {
275
+ if (openedInThinking)
276
+ return returnToThinking();
200
277
  events.push({
201
278
  type: 'toolError',
202
279
  seq: nextSeq(),
@@ -205,8 +282,17 @@ export function createCompletionNormalizer(config) {
205
282
  return;
206
283
  }
207
284
  const { toolCalls, errors } = parseToolCalls(raw, config.tools, dialect);
285
+ if (openedInThinking) {
286
+ if (toolCalls.length === 0)
287
+ return returnToThinking();
288
+ if (closed) {
289
+ heldThinkingCalls = { raw, calls: toolCalls };
290
+ state = 'afterThinkingTool';
291
+ return;
292
+ }
293
+ }
208
294
  for (const call of toolCalls)
209
- emitStreamedToolCall(events, call);
295
+ emitMadeCall(events, call);
210
296
  if (toolCalls.length === 0) {
211
297
  if (errors.length > 0) {
212
298
  for (const error of errors) {
@@ -230,10 +316,7 @@ export function createCompletionNormalizer(config) {
230
316
  }
231
317
  }
232
318
  function abandonToolFrame(events) {
233
- const raw = toolCallBuffer + toolFramingTail;
234
- toolCallBuffer = '';
235
- toolFramingTail = '';
236
- activeToolFrame = null;
319
+ const { raw } = takeToolFrame();
237
320
  state = 'content';
238
321
  events.push({
239
322
  type: 'toolError',
@@ -255,7 +338,113 @@ export function createCompletionNormalizer(config) {
255
338
  return toolFramingTail;
256
339
  case 'contentFraming':
257
340
  return contentFramingTail;
341
+ case 'afterThinkingTool':
342
+ return afterThinkingToolTail;
343
+ }
344
+ }
345
+ // Each step consumes what it can of `buf` in the current state and returns the rest.
346
+ function stepToolFraming(events, buf) {
347
+ if (!activeToolFrame) {
348
+ state = 'content';
349
+ return buf;
350
+ }
351
+ const close = activeToolFrame.close;
352
+ const hit = buf.indexOf(close);
353
+ // `</think>` before the frame closes: the reasoning only mentioned the
354
+ // opener, so the buffered text goes back to reasoning.
355
+ const thinkClose = toolFrameOpenedInThinking ? buf.indexOf(BUILTIN_THINKING.close) : -1;
356
+ if (thinkClose >= 0 && (hit < 0 || thinkClose < hit)) {
357
+ const { raw } = takeToolFrame();
358
+ trimThinkingStart = false;
359
+ emitThinking(events, raw + buf.slice(0, thinkClose));
360
+ closeThinkingBlock(events);
361
+ return buf.slice(thinkClose + BUILTIN_THINKING.close.length);
362
+ }
363
+ if (hit >= 0) {
364
+ toolCallBuffer += buf.slice(0, hit + close.length);
365
+ state = 'content';
366
+ flushToolBuffer(events, true);
367
+ return buf.slice(hit + close.length);
258
368
  }
369
+ const tailLen = trailingPartialMarkerLen(buf, toolFrameOpenedInThinking ? [close, BUILTIN_THINKING.close] : [close]);
370
+ const safeLen = buf.length - tailLen;
371
+ toolCallBuffer += buf.slice(0, safeLen);
372
+ toolFramingTail = buf.slice(safeLen);
373
+ if (toolCallBuffer.length > MAX_TOOL_FRAME_CHARS)
374
+ abandonToolFrame(events);
375
+ return '';
376
+ }
377
+ function closeThinkingBlock(events) {
378
+ for (const { call, made } of deferredCalls)
379
+ if (made)
380
+ emitStreamedToolCall(events, call);
381
+ deferredCalls = [];
382
+ activeThinkingFrame = null;
383
+ trimAfterThinking = contentParts.length === 0;
384
+ state = 'content';
385
+ }
386
+ function releaseHeldCalls(events) {
387
+ const held = heldThinkingCalls;
388
+ heldThinkingCalls = null;
389
+ for (const call of held?.calls ?? [])
390
+ emitMadeCall(events, call);
391
+ }
392
+ // A held call was made if the turn ends, `</think>` closes the block, or
393
+ // another call follows. Any other text means the reasoning quoted it.
394
+ function stepAfterThinkingTool(events, buf) {
395
+ const rest = buf.trimStart();
396
+ const markers = [BUILTIN_THINKING.close, ...toolOpeners];
397
+ const marker = markers.find((m) => rest.startsWith(m));
398
+ if (marker) {
399
+ releaseHeldCalls(events);
400
+ if (marker === BUILTIN_THINKING.close) {
401
+ closeThinkingBlock(events);
402
+ }
403
+ else {
404
+ openToolFrame(marker, markerToEntry.get(marker).spec, true);
405
+ }
406
+ return rest.slice(marker.length);
407
+ }
408
+ if (markers.some((m) => m.startsWith(rest))) {
409
+ afterThinkingToolTail = buf;
410
+ return '';
411
+ }
412
+ const held = heldThinkingCalls;
413
+ heldThinkingCalls = null;
414
+ trimThinkingStart = false;
415
+ emitThinking(events, (held?.raw ?? '') + buf.slice(0, buf.length - rest.length));
416
+ for (const call of held?.calls ?? [])
417
+ deferredCalls.push({ call, made: false });
418
+ undecidedAnswerStart = thinkingParts.length;
419
+ activeThinkingFrame = BUILTIN_THINKING;
420
+ state = 'thinkingFraming';
421
+ return rest;
422
+ }
423
+ function stepThinkingFraming(events, buf) {
424
+ if (!activeThinkingFrame) {
425
+ state = 'content';
426
+ return buf;
427
+ }
428
+ const builtin = activeThinkingFrame === BUILTIN_THINKING;
429
+ const closeMarker = activeThinkingFrame.close;
430
+ const closeIdx = buf.indexOf(closeMarker);
431
+ const toolOpen = builtin ? findEarliestMarker(buf, toolOpeners) : null;
432
+ if (toolOpen && (closeIdx < 0 || toolOpen.idx < closeIdx)) {
433
+ emitThinking(events, buf.slice(0, toolOpen.idx));
434
+ activeThinkingFrame = null;
435
+ openToolFrame(toolOpen.marker, markerToEntry.get(toolOpen.marker).spec, true);
436
+ return buf.slice(toolOpen.idx + toolOpen.marker.length);
437
+ }
438
+ if (closeIdx >= 0) {
439
+ emitThinking(events, buf.slice(0, closeIdx));
440
+ closeThinkingBlock(events);
441
+ return buf.slice(closeIdx + closeMarker.length);
442
+ }
443
+ const tailLen = trailingPartialMarkerLen(buf, builtin ? [closeMarker, ...toolOpeners] : [closeMarker]);
444
+ const safeLen = buf.length - tailLen;
445
+ emitThinking(events, buf.slice(0, safeLen));
446
+ thinkingFramingTail = buf.slice(safeLen);
447
+ return '';
259
448
  }
260
449
  function processSegment(events, text) {
261
450
  let buf = getStateTail() + text;
@@ -263,48 +452,18 @@ export function createCompletionNormalizer(config) {
263
452
  thinkingFramingTail = '';
264
453
  toolFramingTail = '';
265
454
  contentFramingTail = '';
455
+ afterThinkingToolTail = '';
266
456
  while (buf.length > 0) {
267
457
  if (state === 'toolFraming') {
268
- if (!activeToolFrame) {
269
- state = 'content';
270
- continue;
271
- }
272
- const hit = buf.indexOf(activeToolFrame.close);
273
- if (hit >= 0) {
274
- toolCallBuffer += buf.slice(0, hit + activeToolFrame.close.length);
275
- buf = buf.slice(hit + activeToolFrame.close.length);
276
- state = 'content';
277
- flushToolBuffer(events, true);
278
- continue;
279
- }
280
- const tailLen = trailingPartialMarkerLen(buf, [activeToolFrame.close]);
281
- const safeLen = buf.length - tailLen;
282
- toolCallBuffer += buf.slice(0, safeLen);
283
- toolFramingTail = buf.slice(safeLen);
284
- buf = '';
285
- if (toolCallBuffer.length > MAX_TOOL_FRAME_CHARS)
286
- abandonToolFrame(events);
458
+ buf = stepToolFraming(events, buf);
287
459
  continue;
288
460
  }
289
461
  if (state === 'thinkingFraming') {
290
- if (!activeThinkingFrame) {
291
- state = 'content';
292
- continue;
293
- }
294
- const closeMarker = activeThinkingFrame.close;
295
- const closeIdx = buf.indexOf(closeMarker);
296
- if (closeIdx >= 0) {
297
- emitThinking(events, buf.slice(0, closeIdx));
298
- buf = buf.slice(closeIdx + closeMarker.length);
299
- activeThinkingFrame = null;
300
- state = 'content';
301
- continue;
302
- }
303
- const tailLen = trailingPartialMarkerLen(buf, [closeMarker]);
304
- const safeLen = buf.length - tailLen;
305
- emitThinking(events, buf.slice(0, safeLen));
306
- thinkingFramingTail = buf.slice(safeLen);
307
- buf = '';
462
+ buf = stepThinkingFraming(events, buf);
463
+ continue;
464
+ }
465
+ if (state === 'afterThinkingTool') {
466
+ buf = stepAfterThinkingTool(events, buf);
308
467
  continue;
309
468
  }
310
469
  if (state === 'contentFraming') {
@@ -338,13 +497,19 @@ export function createCompletionNormalizer(config) {
338
497
  continue;
339
498
  }
340
499
  const entry = markerToEntry.get(hit.marker);
500
+ // A content frame (Harmony's final channel) carries the answer itself,
501
+ // so the trim carries into it; any other frame ends it.
502
+ if (entry.kind !== 'content')
503
+ trimAfterThinking = false;
341
504
  if (entry.kind === 'tool') {
342
- toolCallBuffer = hit.marker;
343
- activeToolFrame = entry.spec;
344
- state = 'toolFraming';
505
+ openToolFrame(hit.marker, entry.spec, false);
345
506
  }
346
507
  else if (entry.kind === 'thinking') {
347
508
  activeThinkingFrame = entry.spec;
509
+ thinkingBlockStart = thinkingParts.length;
510
+ deferredCalls = [];
511
+ trimThinkingStart = true;
512
+ pendingThinkingWhitespace = '';
348
513
  state = 'thinkingFraming';
349
514
  }
350
515
  else {
@@ -369,10 +534,8 @@ export function createCompletionNormalizer(config) {
369
534
  processSegment(events, token);
370
535
  return events;
371
536
  }
372
- function finish(opts) {
373
- const events = [];
374
- // Drain pending tail bytes so we don't lose trailing chars when the
375
- // stream ends mid-marker.
537
+ // Emits tail bytes held back while a marker might still be arriving.
538
+ function drainTails(events) {
376
539
  if (state === 'content' && contentTail) {
377
540
  emitContent(events, contentTail);
378
541
  contentTail = '';
@@ -386,38 +549,79 @@ export function createCompletionNormalizer(config) {
386
549
  toolFramingTail = '';
387
550
  }
388
551
  else if (state === 'contentFraming' && contentFramingTail) {
389
- // Cutoff inside a partial close marker — emit tail as content.
390
552
  emitContent(events, contentFramingTail);
391
553
  contentFramingTail = '';
392
554
  }
393
- if (state === 'toolFraming' && activeToolFrame) {
394
- if (opts?.error) {
395
- emitContent(events, toolCallBuffer);
396
- toolCallBuffer = '';
397
- activeToolFrame = null;
398
- }
399
- else {
400
- flushToolBuffer(events, false);
401
- }
402
- }
403
- else if (state === 'contentFraming') {
555
+ }
556
+ // A frame still open at the end is parsed, or returned as text on error.
557
+ function closeOpenFrames(events, failed) {
558
+ if (state === 'contentFraming') {
404
559
  activeContentFrame = null;
405
560
  }
561
+ else if (state === 'toolFraming' && activeToolFrame && !failed) {
562
+ flushToolBuffer(events, false);
563
+ }
564
+ else if (state === 'toolFraming' && activeToolFrame) {
565
+ const { raw, openedInThinking } = takeToolFrame();
566
+ if (openedInThinking)
567
+ emitThinking(events, raw);
568
+ else
569
+ emitContent(events, raw);
570
+ }
571
+ }
572
+ // The turn ended right after a call inside `<think>`: it was made, unless
573
+ // the turn failed. A partial marker buffered after it stays reasoning.
574
+ function settleHeldCalls(events, failed) {
575
+ if (!heldThinkingCalls)
576
+ return;
577
+ if (failed) {
578
+ emitThinking(events, heldThinkingCalls.raw + afterThinkingToolTail);
579
+ heldThinkingCalls = null;
580
+ }
581
+ else {
582
+ releaseHeldCalls(events);
583
+ if (afterThinkingToolTail.trim())
584
+ emitThinking(events, afterThinkingToolTail);
585
+ }
586
+ afterThinkingToolTail = '';
587
+ }
588
+ // The model stopped on its own inside the prompt-opened `<think>`: calls it
589
+ // left there were made, and the reasoning after them, or the whole block
590
+ // when the turn has no call at all, is the answer.
591
+ function promoteOpenThinking(events, callsLeftInBlock, finalCalls) {
592
+ if (contentParts.length > 0)
593
+ return;
594
+ if (callsLeftInBlock) {
595
+ emitContent(events, thinkingParts.slice(undecidedAnswerStart).join('').trim());
596
+ }
597
+ else if (streamedToolCallKeys.size + finalCalls.length === 0) {
598
+ emitContent(events, thinkingParts.slice(thinkingBlockStart).join('').trim());
599
+ }
600
+ }
601
+ function finish(opts) {
602
+ const events = [];
603
+ const failed = !!opts?.error;
604
+ const finalCalls = failed ? [] : (opts?.toolCalls ?? []);
605
+ const naturalStop = opts?.stopReason === 'eos' || opts?.stopReason === 'stopSequence';
606
+ drainTails(events);
607
+ closeOpenFrames(events, failed);
608
+ settleHeldCalls(events, failed);
609
+ // The block never closed, so every call left in it was made. A cancelled
610
+ // turn says nothing about the ones the model had not confirmed.
611
+ const cancelled = opts?.stopReason === 'cancelled';
612
+ const leftInBlock = failed ? [] : deferredCalls.filter((d) => d.made || !cancelled);
613
+ deferredCalls = [];
614
+ for (const { call } of leftInBlock)
615
+ emitStreamedToolCall(events, call);
616
+ if (activeThinkingFrame === BUILTIN_THINKING && naturalStop && !failed) {
617
+ promoteOpenThinking(events, leftInBlock.length > 0, finalCalls);
618
+ }
406
619
  activeThinkingFrame = null;
407
620
  state = 'content';
408
- if (!opts?.error) {
409
- if (opts?.toolCalls) {
410
- for (const call of opts.toolCalls) {
411
- emitFinalToolCall(events, call);
412
- }
413
- }
414
- }
621
+ for (const call of finalCalls)
622
+ emitFinalToolCall(events, call);
415
623
  if (opts?.stats) {
416
- events.push({
417
- type: 'completionStats',
418
- seq: nextSeq(),
419
- stats: opts.stats
420
- });
624
+ events.push({ type: 'completionStats', seq: nextSeq(), stats: opts.stats });
421
625
  }
422
626
  const raw = { fullText: rawText };
423
627
  if (opts?.error) {
@@ -430,12 +634,7 @@ export function createCompletionNormalizer(config) {
430
634
  });
431
635
  }
432
636
  else if (opts?.stopReason) {
433
- events.push({
434
- type: 'completionDone',
435
- seq: nextSeq(),
436
- stopReason: opts.stopReason,
437
- raw
438
- });
637
+ events.push({ type: 'completionDone', seq: nextSeq(), stopReason: opts.stopReason, raw });
439
638
  }
440
639
  else {
441
640
  events.push({ type: 'completionDone', seq: nextSeq(), raw });
@@ -0,0 +1,7 @@
1
+ export declare const ARCHITECTURE_SCAN_BYTES: number;
2
+ /**
3
+ * The `general.architecture` value from a GGUF header, read from at most the
4
+ * first `ARCHITECTURE_SCAN_BYTES` of the file. Undefined when the file is
5
+ * unreadable, is not GGUF, or does not name an architecture in that range.
6
+ */
7
+ export declare function readGgufArchitecture(filePath: string): Promise<string | undefined>;
@@ -0,0 +1,142 @@
1
+ import fs from 'bare-fs';
2
+ import Buffer from 'bare-buffer';
3
+ const GGUF_MAGIC = 0x46554747;
4
+ const STRING_TYPE = 8;
5
+ const ARRAY_TYPE = 9;
6
+ const FIXED_SIZES = {
7
+ 0: 1,
8
+ 1: 1,
9
+ 2: 2,
10
+ 3: 2,
11
+ 4: 4,
12
+ 5: 4,
13
+ 6: 4,
14
+ 7: 1,
15
+ 10: 8,
16
+ 11: 8,
17
+ 12: 8
18
+ };
19
+ // GGUF writers put `general.architecture` first, ahead of the tokenizer arrays
20
+ // that make a header large, so a key past this point is not worth the read.
21
+ export const ARCHITECTURE_SCAN_BYTES = 1024 * 1024;
22
+ /**
23
+ * Walks the header in the first `end` bytes of `buffer` to
24
+ * `general.architecture`. Undefined when the header is not GGUF, holds a value
25
+ * type it cannot skip, or does not reach the key within those bytes.
26
+ */
27
+ function scanForArchitecture(buffer, end) {
28
+ let offset = 0;
29
+ let bigEndian = false;
30
+ const fits = (bytes) => offset + bytes <= end;
31
+ function u32() {
32
+ if (!fits(4))
33
+ return undefined;
34
+ const value = bigEndian ? buffer.readUInt32BE(offset) : buffer.readUInt32LE(offset);
35
+ offset += 4;
36
+ return value;
37
+ }
38
+ function u64() {
39
+ if (!fits(8))
40
+ return undefined;
41
+ const value = bigEndian ? buffer.readBigUInt64BE(offset) : buffer.readBigUInt64LE(offset);
42
+ offset += 8;
43
+ return Number(value);
44
+ }
45
+ function skip(bytes) {
46
+ if (!fits(bytes))
47
+ return false;
48
+ offset += bytes;
49
+ return true;
50
+ }
51
+ function string() {
52
+ const length = u64();
53
+ if (length === undefined || !fits(length))
54
+ return undefined;
55
+ const value = buffer.toString('utf8', offset, offset + length);
56
+ offset += length;
57
+ return value;
58
+ }
59
+ function skipString() {
60
+ const length = u64();
61
+ return length !== undefined && skip(length);
62
+ }
63
+ function skipValue(type) {
64
+ const size = FIXED_SIZES[type];
65
+ if (size !== undefined)
66
+ return skip(size);
67
+ if (type === STRING_TYPE)
68
+ return skipString();
69
+ if (type !== ARRAY_TYPE)
70
+ return false;
71
+ const elementType = u32();
72
+ const count = u64();
73
+ if (elementType === undefined || count === undefined)
74
+ return false;
75
+ const elementSize = FIXED_SIZES[elementType];
76
+ if (elementSize !== undefined)
77
+ return skip(count * elementSize);
78
+ if (elementType !== STRING_TYPE)
79
+ return false;
80
+ for (let i = 0; i < count; i++) {
81
+ if (!skipString())
82
+ return false;
83
+ }
84
+ return true;
85
+ }
86
+ if (u32() !== GGUF_MAGIC)
87
+ return undefined;
88
+ // The magic is the same bytes in both byte orders. A big-endian header is
89
+ // told apart by its version, whose low 16 bits read 0 as little-endian.
90
+ let version = u32();
91
+ if (version === undefined)
92
+ return undefined;
93
+ if ((version & 0xffff) === 0) {
94
+ bigEndian = true;
95
+ version = buffer.readUInt32BE(4);
96
+ }
97
+ if (version !== 2 && version !== 3)
98
+ return undefined;
99
+ if (!skip(8))
100
+ return undefined;
101
+ const kvCount = u64();
102
+ if (kvCount === undefined)
103
+ return undefined;
104
+ for (let i = 0; i < kvCount; i++) {
105
+ const key = string();
106
+ const type = u32();
107
+ if (key === undefined || type === undefined)
108
+ return undefined;
109
+ if (key === 'general.architecture' && type === STRING_TYPE)
110
+ return string();
111
+ if (!skipValue(type))
112
+ return undefined;
113
+ }
114
+ return undefined;
115
+ }
116
+ /**
117
+ * The `general.architecture` value from a GGUF header, read from at most the
118
+ * first `ARCHITECTURE_SCAN_BYTES` of the file. Undefined when the file is
119
+ * unreadable, is not GGUF, or does not name an architecture in that range.
120
+ */
121
+ export async function readGgufArchitecture(filePath) {
122
+ let fd;
123
+ try {
124
+ fd = await fs.open(filePath, 'r');
125
+ const buffer = Buffer.alloc(ARCHITECTURE_SCAN_BYTES);
126
+ let length = 0;
127
+ while (length < buffer.length) {
128
+ const read = await fs.read(fd, buffer, length, buffer.length - length, length);
129
+ if (read === 0)
130
+ break;
131
+ length += read;
132
+ }
133
+ return scanForArchitecture(buffer, length);
134
+ }
135
+ catch {
136
+ return undefined;
137
+ }
138
+ finally {
139
+ if (fd !== undefined)
140
+ await fs.close(fd).catch(() => { });
141
+ }
142
+ }
@@ -1,9 +1,3 @@
1
- /**
2
- * Extract tensor names from all GGUF shards and write to {baseFilename}.tensors.txt
3
- * Required for sharded models to enable incremental/async loading
4
- * @throws ModelLoadFailedError if any shard fails extraction
5
- */
6
- export declare function extractAndWriteTensorsFile(shardDir: string, shardFilenames: string[], baseFilename: string): Promise<string>;
7
1
  /**
8
2
  * Check if a file has a valid GGUF header structure.
9
3
  * Returns true if the header parses successfully and contains tensor metadata.