@qvac/inference 0.20.1 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/NOTICE +102 -40
  2. package/README.md +8 -5
  3. package/dist/api/bci-transcribe.d.ts +4 -1
  4. package/dist/api/index.d.ts +1 -0
  5. package/dist/api/index.js +1 -0
  6. package/dist/api/rpc-server.d.ts +19 -0
  7. package/dist/api/rpc-server.js +44 -0
  8. package/dist/api/text-to-speech.d.ts +9 -0
  9. package/dist/api/text-to-speech.js +9 -0
  10. package/dist/api/transcribe.d.ts +11 -7
  11. package/dist/api/transcribe.js +47 -15
  12. package/dist/api/video-request.d.ts +5 -5
  13. package/dist/dispatch.js +39 -7
  14. package/dist/errors/index.d.ts +7 -0
  15. package/dist/errors/index.js +15 -0
  16. package/dist/handlers/assess-model-fit.d.ts +8 -20
  17. package/dist/handlers/assess-model-fit.js +195 -77
  18. package/dist/handlers/completion-orchestrate.js +17 -0
  19. package/dist/handlers/get-loaded-model-info.js +4 -1
  20. package/dist/handlers/get-model-info.js +29 -1
  21. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  22. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  23. package/dist/handlers/load-model/hyperdrive.js +2 -2
  24. package/dist/handlers/load-model/registry.js +59 -0
  25. package/dist/handlers/rpc-server.d.ts +23 -0
  26. package/dist/handlers/rpc-server.js +141 -0
  27. package/dist/index.d.ts +2 -1
  28. package/dist/index.js +2 -1
  29. package/dist/models/registry/models.d.ts +7731 -5672
  30. package/dist/models/registry/models.js +4652 -2526
  31. package/dist/models/registry/resource-profiles.js +481 -94
  32. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  34. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  36. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  40. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
  41. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  42. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  43. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  44. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  45. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  46. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  47. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  48. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  49. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  50. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  51. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  52. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  53. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  54. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  55. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  56. package/dist/plugins/ops/bci-transcribe.js +18 -13
  57. package/dist/plugins/ops/load-model.js +2 -2
  58. package/dist/plugins/ops/transcribe.d.ts +4 -2
  59. package/dist/plugins/ops/transcribe.js +39 -2
  60. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  61. package/dist/plugins/ops/translate-stats.js +64 -7
  62. package/dist/plugins/ops/translate.js +20 -4
  63. package/dist/profiling/operation-metrics.js +56 -0
  64. package/dist/registry.js +4 -0
  65. package/dist/resources/model-fit/assess.d.ts +37 -2
  66. package/dist/resources/model-fit/assess.js +24 -3
  67. package/dist/resources/model-fit/estimators/llm.js +2 -2
  68. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  69. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
  70. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
  71. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  72. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  73. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  74. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  75. package/dist/resources/model-fit/floor.js +2 -2
  76. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  77. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  78. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  79. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  80. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  81. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  82. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  83. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  84. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  85. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  86. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  87. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  88. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  89. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  90. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  91. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  92. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  93. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  94. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  95. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  96. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  97. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  98. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  99. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  100. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  101. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  102. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  103. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  104. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  105. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  106. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  107. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  108. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  109. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  110. package/dist/resources/model-fit/platform.d.ts +9 -0
  111. package/dist/resources/model-fit/platform.js +28 -0
  112. package/dist/rpc/device-map.d.ts +13 -0
  113. package/dist/rpc/device-map.js +22 -0
  114. package/dist/rpc/discovery.d.ts +12 -0
  115. package/dist/rpc/discovery.js +225 -0
  116. package/dist/rpc/ggml-provider.d.ts +4 -0
  117. package/dist/rpc/ggml-provider.js +25 -0
  118. package/dist/rpc/index.d.ts +2 -0
  119. package/dist/rpc/index.js +1 -0
  120. package/dist/rpc/instance.d.ts +6 -0
  121. package/dist/rpc/instance.js +21 -0
  122. package/dist/rpc/inventory.d.ts +5 -0
  123. package/dist/rpc/inventory.js +144 -0
  124. package/dist/rpc/manager.d.ts +14 -0
  125. package/dist/rpc/manager.js +174 -0
  126. package/dist/rpc/network.d.ts +11 -0
  127. package/dist/rpc/network.js +85 -0
  128. package/dist/rpc/provider.d.ts +6 -0
  129. package/dist/rpc/provider.js +24 -0
  130. package/dist/runtime/lifecycle.js +31 -13
  131. package/dist/runtime/request-context.d.ts +1 -1
  132. package/dist/runtime/state.js +5 -0
  133. package/dist/schemas/assess-model-fit.d.ts +269 -85
  134. package/dist/schemas/assess-model-fit.js +78 -54
  135. package/dist/schemas/audio-gen.d.ts +6 -6
  136. package/dist/schemas/audio-gen.js +2 -2
  137. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  138. package/dist/schemas/batch-completion-stream.js +5 -3
  139. package/dist/schemas/bci-config.js +1 -1
  140. package/dist/schemas/bci.d.ts +86 -1
  141. package/dist/schemas/bci.js +21 -2
  142. package/dist/schemas/cancel.d.ts +10 -0
  143. package/dist/schemas/cancel.js +2 -0
  144. package/dist/schemas/common.d.ts +708 -66
  145. package/dist/schemas/common.js +7 -0
  146. package/dist/schemas/completion-event.d.ts +6 -6
  147. package/dist/schemas/completion-stream.d.ts +27 -8
  148. package/dist/schemas/completion-stream.js +41 -4
  149. package/dist/schemas/config.d.ts +104 -36
  150. package/dist/schemas/config.js +10 -0
  151. package/dist/schemas/embed.d.ts +2 -2
  152. package/dist/schemas/error.js +13 -0
  153. package/dist/schemas/errors.d.ts +1 -0
  154. package/dist/schemas/errors.js +5 -0
  155. package/dist/schemas/index.d.ts +1 -0
  156. package/dist/schemas/index.js +1 -0
  157. package/dist/schemas/llamacpp-config.d.ts +25 -18
  158. package/dist/schemas/llamacpp-config.js +26 -16
  159. package/dist/schemas/load-model.d.ts +2608 -469
  160. package/dist/schemas/load-model.js +2 -2
  161. package/dist/schemas/mcp-adapter.d.ts +4 -0
  162. package/dist/schemas/model-config-schemas.d.ts +183 -6
  163. package/dist/schemas/model-info.d.ts +56 -0
  164. package/dist/schemas/model-info.js +5 -1
  165. package/dist/schemas/plugin.d.ts +8 -2
  166. package/dist/schemas/profiling.d.ts +4 -4
  167. package/dist/schemas/reload-config.d.ts +30 -0
  168. package/dist/schemas/rpc-server.d.ts +101 -0
  169. package/dist/schemas/rpc-server.js +87 -0
  170. package/dist/schemas/sdcpp-config.d.ts +7 -3
  171. package/dist/schemas/sdcpp-config.js +37 -2
  172. package/dist/schemas/system-resources.d.ts +4 -4
  173. package/dist/schemas/text-to-speech.d.ts +529 -1
  174. package/dist/schemas/text-to-speech.js +122 -12
  175. package/dist/schemas/tools.d.ts +2 -0
  176. package/dist/schemas/tools.js +8 -0
  177. package/dist/schemas/transcription-config.d.ts +5 -0
  178. package/dist/schemas/transcription-config.js +20 -2
  179. package/dist/schemas/transcription.d.ts +164 -0
  180. package/dist/schemas/transcription.js +71 -4
  181. package/dist/schemas/vla.d.ts +1 -1
  182. package/dist/surface.d.ts +4 -1
  183. package/dist/surface.js +3 -1
  184. package/dist/utils/addon-responses.d.ts +5 -1
  185. package/dist/utils/archive.js +5 -1
  186. package/dist/utils/asr-diagnostics.d.ts +12 -0
  187. package/dist/utils/asr-diagnostics.js +42 -0
  188. package/dist/utils/asr-events.d.ts +1 -0
  189. package/dist/utils/asr-events.js +3 -2
  190. package/dist/utils/cache/paths.js +3 -2
  191. package/dist/utils/mcp-adapter.js +7 -1
  192. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  193. package/dist/utils/multi-gpu-mobile.js +8 -6
  194. package/dist/utils/path-security.d.ts +5 -0
  195. package/dist/utils/path-security.js +14 -0
  196. package/dist/utils/tool-helpers.d.ts +4 -0
  197. package/dist/utils/tool-helpers.js +2 -0
  198. package/dist/utils/tools/defer.d.ts +93 -0
  199. package/dist/utils/tools/defer.js +290 -0
  200. package/dist/utils/tools/index.d.ts +1 -0
  201. package/dist/utils/tools/index.js +1 -0
  202. package/dist/utils/transcribe-metadata.d.ts +21 -7
  203. package/dist/utils/transcribe-metadata.js +24 -8
  204. package/dist/utils/tts-stats.d.ts +6 -0
  205. package/dist/utils/tts-stats.js +14 -1
  206. package/package.json +43 -29
  207. package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
  208. package/dist/resources/model-fit/fit-stub/index.js +0 -2
  209. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
  210. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
  211. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
@@ -0,0 +1,290 @@
1
+ import { validateTools } from '../tool-helpers.js';
2
+ /**
3
+ * Reserved name of the built-in search tool. A caller cannot register a tool
4
+ * under this name, and it is never itself deferred — the model needs one
5
+ * callable entry point to reach everything else.
6
+ */
7
+ export const TOOL_SEARCH_NAME = 'tool_search';
8
+ const DEFAULT_SEARCH_LIMIT = 3;
9
+ const MAX_SEARCH_LIMIT = 8;
10
+ // Ceiling on the definitions one search may append. A query matching a dozen
11
+ // large schemas would otherwise undo the saving the catalog just made.
12
+ const MAX_DEFINITION_BYTES = 8 * 1024;
13
+ // Fixed text, so a repeated no-match search stays byte-identical and cannot
14
+ // move the KV divergence point.
15
+ const NO_MATCH_HINT = 'No tool matched. Search again with a different capability word, or answer without a tool.';
16
+ export function isDeferred(tool) {
17
+ return tool.deferLoading === true;
18
+ }
19
+ export function partitionTools(tools) {
20
+ const alwaysLoaded = [];
21
+ const deferred = [];
22
+ for (const tool of tools) {
23
+ if (isDeferred(tool))
24
+ deferred.push(tool);
25
+ else
26
+ alwaysLoaded.push(tool);
27
+ }
28
+ return { alwaysLoaded, deferred };
29
+ }
30
+ /** Strip the registration-only fields so what reaches the model is a plain definition. */
31
+ export function toWireTool(tool) {
32
+ const { deferLoading: _deferLoading, group: _group, ...wire } = tool;
33
+ return wire;
34
+ }
35
+ /**
36
+ * Compact listing of the deferred inventory: one line per tool, name and
37
+ * description only, grouped by `group` where present.
38
+ *
39
+ * Caller order is preserved and groups appear in first-use order, because this
40
+ * text sits in the prompt prefix — a reordering between turns would move the
41
+ * divergence point and cost a re-prefill.
42
+ */
43
+ export function buildCatalog(deferred) {
44
+ const ungrouped = [];
45
+ const groups = new Map();
46
+ for (const tool of deferred) {
47
+ if (tool.group === undefined) {
48
+ ungrouped.push(tool);
49
+ continue;
50
+ }
51
+ const existing = groups.get(tool.group);
52
+ if (existing)
53
+ existing.push(tool);
54
+ else
55
+ groups.set(tool.group, [tool]);
56
+ }
57
+ const lines = [];
58
+ const describe = (tool) => `- ${tool.name}: ${tool.description}`;
59
+ for (const tool of ungrouped)
60
+ lines.push(describe(tool));
61
+ for (const [group, tools] of groups) {
62
+ lines.push(`[${group}]`);
63
+ for (const tool of tools)
64
+ lines.push(describe(tool));
65
+ }
66
+ return lines.join('\n');
67
+ }
68
+ /**
69
+ * The built-in search tool, carrying the catalog in its description.
70
+ *
71
+ * The catalog is deliberately not a set of parameterless tool entries: those
72
+ * render into the same native block and the grammar would accept a call to a
73
+ * tool whose schema the model has never seen. Description text cannot be
74
+ * called, which is the whole point.
75
+ */
76
+ export function buildToolSearchTool(deferred) {
77
+ return {
78
+ type: 'function',
79
+ name: TOOL_SEARCH_NAME,
80
+ description: 'Find tools by capability and load their full definitions into the conversation. ' +
81
+ 'The tools below are registered but their parameter schemas are not here yet — search ' +
82
+ 'for one by capability or by exact name, then call it on your next step.\n\n' +
83
+ `Available tools:\n${buildCatalog(deferred)}`,
84
+ parameters: {
85
+ type: 'object',
86
+ properties: {
87
+ query: {
88
+ type: 'string',
89
+ description: 'A capability to search for, or the exact name of a tool to load.'
90
+ },
91
+ limit: {
92
+ type: 'integer',
93
+ description: `Maximum definitions to load (default ${DEFAULT_SEARCH_LIMIT}, max ${MAX_SEARCH_LIMIT}).`
94
+ }
95
+ },
96
+ required: ['query']
97
+ }
98
+ };
99
+ }
100
+ function tokenize(text) {
101
+ return text
102
+ .toLowerCase()
103
+ .split(/[^a-z0-9]+/)
104
+ .filter((token) => token.length > 1);
105
+ }
106
+ /**
107
+ * Whether a query word matches a haystack word. A prefix so "repo" finds
108
+ * "repository", but anchored at a word start: a bare substring test scores
109
+ * "at" against "create_issue" and every query ends up matching everything.
110
+ */
111
+ function matchesWord(words, token) {
112
+ return words.some((word) => word.startsWith(token));
113
+ }
114
+ function scoreTool(tool, query, tokens) {
115
+ const name = tool.name.toLowerCase();
116
+ const normalizedQuery = query.trim().toLowerCase();
117
+ // Exact name wins outright: the model asking for a tool by name is not a
118
+ // capability search and must not be outranked by a wordier description.
119
+ if (name === normalizedQuery)
120
+ return 1000;
121
+ let score = 0;
122
+ if (normalizedQuery.length > 2 && name.includes(normalizedQuery))
123
+ score += 100;
124
+ const nameWords = tokenize(tool.name);
125
+ const descriptionWords = tokenize(tool.description);
126
+ const groupWords = tokenize(tool.group ?? '');
127
+ for (const token of tokens) {
128
+ if (matchesWord(nameWords, token))
129
+ score += 50;
130
+ if (matchesWord(descriptionWords, token))
131
+ score += 10;
132
+ if (matchesWord(groupWords, token))
133
+ score += 5;
134
+ }
135
+ return score;
136
+ }
137
+ /**
138
+ * Rank the deferred inventory against a query. Exact name first, then text
139
+ * match. Ties keep registration order so repeated searches are deterministic.
140
+ */
141
+ export function searchDeferredTools(deferred, query, limit) {
142
+ const tokens = tokenize(query);
143
+ const scored = deferred
144
+ .map((tool, index) => ({ tool, index, score: scoreTool(tool, query, tokens) }))
145
+ .filter((entry) => entry.score > 0)
146
+ .sort((a, b) => b.score - a.score || a.index - b.index);
147
+ const cap = Math.min(Math.max(Math.trunc(limit ?? DEFAULT_SEARCH_LIMIT), 1), MAX_SEARCH_LIMIT);
148
+ const picked = [];
149
+ let bytes = 0;
150
+ for (const { tool } of scored) {
151
+ if (picked.length >= cap)
152
+ break;
153
+ const size = JSON.stringify(toWireTool(tool)).length;
154
+ // Always let the top match through, however large; past that, stop rather
155
+ // than truncate, so what the model receives is always a complete schema.
156
+ if (picked.length > 0 && bytes + size > MAX_DEFINITION_BYTES)
157
+ break;
158
+ picked.push(tool);
159
+ bytes += size;
160
+ }
161
+ return picked;
162
+ }
163
+ export function buildToolSearchResult(query, matches, alreadyLoaded) {
164
+ const loaded = matches.map((tool) => tool.name);
165
+ const result = {
166
+ tool_search: {
167
+ query,
168
+ loaded,
169
+ already_loaded: [...alreadyLoaded],
170
+ // Without this the model reads an empty result as "try again" and burns
171
+ // rounds rephrasing the same query.
172
+ ...(loaded.length === 0 && alreadyLoaded.length === 0 ? { hint: NO_MATCH_HINT } : {})
173
+ },
174
+ tools: matches.map(toWireTool)
175
+ };
176
+ return JSON.stringify(result);
177
+ }
178
+ function readSearchResult(content) {
179
+ if (!content.includes(`"${TOOL_SEARCH_NAME}"`))
180
+ return null;
181
+ try {
182
+ const parsed = JSON.parse(content);
183
+ if (typeof parsed !== 'object' || parsed === null)
184
+ return null;
185
+ const envelope = parsed[TOOL_SEARCH_NAME];
186
+ if (typeof envelope !== 'object' || envelope === null)
187
+ return null;
188
+ const loaded = envelope['loaded'];
189
+ if (!Array.isArray(loaded))
190
+ return null;
191
+ return {
192
+ query: '',
193
+ loaded: loaded.filter((name) => typeof name === 'string'),
194
+ already_loaded: []
195
+ };
196
+ }
197
+ catch {
198
+ // A tool result that isn't ours, or isn't JSON at all.
199
+ return null;
200
+ }
201
+ }
202
+ /**
203
+ * Which deferred definitions this conversation already carries.
204
+ *
205
+ * Read back from the history rather than held in the session, so `completion()`
206
+ * stays stateless: a reopened chat replaying the same messages resolves the
207
+ * same loaded set, and two concurrent conversations cannot see each other's.
208
+ */
209
+ export function loadedToolNames(history) {
210
+ const loaded = new Set();
211
+ for (const message of history) {
212
+ if (message.role !== 'tool')
213
+ continue;
214
+ const envelope = readSearchResult(message.content);
215
+ if (!envelope)
216
+ continue;
217
+ for (const name of envelope.loaded)
218
+ loaded.add(name);
219
+ }
220
+ return loaded;
221
+ }
222
+ /**
223
+ * Split a registered tool list into what the prompt carries and what the
224
+ * parser accepts.
225
+ *
226
+ * Loaded definitions stay out of `toolsToRender` on purpose. They live in the
227
+ * history as tool results, which appends to the prefix instead of rewriting
228
+ * the block at the front of it — a tool loaded mid-conversation must not cost
229
+ * a re-prefill of everything before it.
230
+ *
231
+ * Returns `null` when nothing defers, so the existing path is untouched.
232
+ */
233
+ export function resolveDeferredTools(tools, history) {
234
+ if (!tools || tools.length === 0)
235
+ return null;
236
+ const { alwaysLoaded, deferred } = partitionTools(tools);
237
+ if (deferred.length === 0)
238
+ return null;
239
+ const loaded = loadedToolNames(history);
240
+ const rendered = [...alwaysLoaded.map(toWireTool), buildToolSearchTool(deferred)];
241
+ return {
242
+ toolsToRender: rendered,
243
+ callableTools: [
244
+ ...rendered,
245
+ ...deferred.filter((tool) => loaded.has(tool.name)).map(toWireTool)
246
+ ],
247
+ deferred: [...deferred]
248
+ };
249
+ }
250
+ /**
251
+ * Generation params for a turn whose history may have loaded deferred definitions.
252
+ *
253
+ * The addon builds its tool-call grammar from the rendered tools only, and a
254
+ * loaded definition lives in the history rather than that block, so under the
255
+ * default grammar the model can never produce the name it just loaded. Unless
256
+ * the caller asked for more than the default, those turns run with the grammar
257
+ * off and the parser, which does accept loaded names, validates the call.
258
+ */
259
+ export function withDeferredToolChoice(generationParams, resolution) {
260
+ const toolChoice = generationParams?.tool_choice;
261
+ // An explicit "auto" is the default, which is exactly the grammar that cannot reach a loaded tool.
262
+ if (toolChoice !== undefined && toolChoice !== 'auto')
263
+ return generationParams;
264
+ if (!resolution || resolution.callableTools.length === resolution.toolsToRender.length) {
265
+ return generationParams;
266
+ }
267
+ return { ...(generationParams ?? {}), tool_choice: 'none' };
268
+ }
269
+ /**
270
+ * Run a `tool_search` call and produce the `tool` message that carries its
271
+ * result. Pure over the history it is given; the caller appends the message.
272
+ *
273
+ * Takes either wire `Tool`s or the Zod-schema `ToolInput` form, because what a
274
+ * caller holds is whatever they passed to `completion()`. A `ToolInput` whose
275
+ * `parameters` is still a Zod object would otherwise be serialised as Zod
276
+ * internals and reach the model as a definition it cannot call.
277
+ */
278
+ export function executeToolSearch(tools, args, history) {
279
+ const { tools: wire } = validateTools([...tools]);
280
+ const { deferred } = partitionTools(wire);
281
+ const query = typeof args['query'] === 'string' ? args['query'] : '';
282
+ const limit = typeof args['limit'] === 'number' ? args['limit'] : undefined;
283
+ const loaded = loadedToolNames(history);
284
+ const matches = searchDeferredTools(deferred, query, limit);
285
+ // A definition already in the history is not appended a second time; naming
286
+ // it back tells the model it can call the tool without searching again.
287
+ const alreadyLoaded = matches.filter((tool) => loaded.has(tool.name)).map((tool) => tool.name);
288
+ const fresh = matches.filter((tool) => !loaded.has(tool.name));
289
+ return buildToolSearchResult(query, fresh, alreadyLoaded);
290
+ }
@@ -1,2 +1,3 @@
1
1
  export { detectToolDialectFromName } from './dialect.js';
2
2
  export { parseToolCalls } from './parser.js';
3
+ export { TOOL_SEARCH_NAME, buildCatalog, buildToolSearchTool, executeToolSearch, loadedToolNames, partitionTools, resolveDeferredTools, searchDeferredTools } from './defer.js';
@@ -1,2 +1,3 @@
1
1
  export { detectToolDialectFromName } from './dialect.js';
2
2
  export { parseToolCalls } from './parser.js';
3
+ export { TOOL_SEARCH_NAME, buildCatalog, buildToolSearchTool, executeToolSearch, loadedToolNames, partitionTools, resolveDeferredTools, searchDeferredTools } from './defer.js';
@@ -1,20 +1,34 @@
1
1
  import { type TranscribeSegment } from '../schemas/index.js';
2
- export interface WhisperAddonSegment {
2
+ /**
3
+ * A native transcript segment, as emitted by either engine of
4
+ * `@qvac/asr-ggml` and by `@qvac/bci-whispercpp`. `text` plus the timing
5
+ * fields are shared; `isEndOfTurn` and `startsWord` are parakeet-only, and
6
+ * `windowStartTimestep` is BCI delta-streaming-only.
7
+ */
8
+ export interface AsrAddonSegment {
3
9
  text: string;
4
10
  start?: number;
5
11
  end?: number;
6
12
  toAppend?: boolean;
7
13
  id?: number;
14
+ /** Parakeet: the segment ends on a recognized end-of-utterance boundary. */
15
+ isEndOfTurn?: boolean;
16
+ /** Parakeet: the segment begins a new SentencePiece word. */
17
+ startsWord?: boolean;
18
+ /** BCI delta streaming: absolute timestep at which the owning window began. */
19
+ windowStartTimestep?: number;
8
20
  }
9
21
  /**
10
- * Normalize a native whisper-addon segment to the engine-level `TranscribeSegment`
11
- * shape exposed to callers: seconds → milliseconds, `toAppend` → `append`,
12
- * and defaults for optional fields.
22
+ * Normalize a native transcript segment to the engine-level
23
+ * `TranscribeSegment` shape exposed to callers: seconds → milliseconds,
24
+ * `toAppend` → `append`, and defaults for optional fields. The
25
+ * engine-specific fields are carried through only when the engine sent them,
26
+ * so a segment does not gain fields whose value would be meaningless.
13
27
  */
14
- export declare function toTranscribeSegment(chunk: WhisperAddonSegment): TranscribeSegment;
28
+ export declare function toTranscribeSegment(chunk: AsrAddonSegment): TranscribeSegment & {
29
+ windowStartTimestep?: number;
30
+ };
15
31
  /**
16
32
  * Guard used by transcription ops when the caller opts into `metadata: true`.
17
- * Only the whisper engine emits per-segment metadata; any other engine
18
- * requesting metadata is rejected with `TranscriptionFailedError`.
19
33
  */
20
34
  export declare function assertMetadataSupported(modelId: string, engineType: string, metadata: boolean | undefined): void;
@@ -1,9 +1,11 @@
1
1
  import { ModelType } from '../schemas/index.js';
2
2
  import { TranscriptionFailedError } from '../errors/index.js';
3
3
  /**
4
- * Normalize a native whisper-addon segment to the engine-level `TranscribeSegment`
5
- * shape exposed to callers: seconds → milliseconds, `toAppend` → `append`,
6
- * and defaults for optional fields.
4
+ * Normalize a native transcript segment to the engine-level
5
+ * `TranscribeSegment` shape exposed to callers: seconds → milliseconds,
6
+ * `toAppend` → `append`, and defaults for optional fields. The
7
+ * engine-specific fields are carried through only when the engine sent them,
8
+ * so a segment does not gain fields whose value would be meaningless.
7
9
  */
8
10
  export function toTranscribeSegment(chunk) {
9
11
  return {
@@ -11,18 +13,32 @@ export function toTranscribeSegment(chunk) {
11
13
  startMs: (chunk.start ?? 0) * 1000,
12
14
  endMs: (chunk.end ?? 0) * 1000,
13
15
  append: chunk.toAppend ?? false,
14
- id: chunk.id ?? 0
16
+ id: chunk.id ?? 0,
17
+ ...(chunk.isEndOfTurn !== undefined && { isEndOfTurn: chunk.isEndOfTurn }),
18
+ ...(chunk.startsWord !== undefined && { startsWord: chunk.startsWord }),
19
+ ...(chunk.windowStartTimestep !== undefined && {
20
+ windowStartTimestep: chunk.windowStartTimestep
21
+ })
15
22
  };
16
23
  }
24
+ /**
25
+ * Engines whose native layer emits per-segment metadata. Both `@qvac/asr-ggml`
26
+ * engines do: the parakeet output serializer sends `start`, `end`, `id`,
27
+ * `toAppend`, `isEndOfTurn` and `startsWord` for every segment, and
28
+ * `timestampsEnabled` defaults to true.
29
+ */
30
+ const METADATA_CAPABLE_ENGINES = [
31
+ ModelType.whispercppTranscription,
32
+ ModelType.parakeetTranscription
33
+ ];
17
34
  /**
18
35
  * Guard used by transcription ops when the caller opts into `metadata: true`.
19
- * Only the whisper engine emits per-segment metadata; any other engine
20
- * requesting metadata is rejected with `TranscriptionFailedError`.
21
36
  */
22
37
  export function assertMetadataSupported(modelId, engineType, metadata) {
23
38
  if (!metadata)
24
39
  return;
25
- if (engineType !== ModelType.whispercppTranscription) {
26
- throw new TranscriptionFailedError(`metadata mode is not supported on model ${modelId} (engine: ${engineType || 'unknown'}); only the whisper engine supports metadata`);
40
+ if (!METADATA_CAPABLE_ENGINES.includes(engineType)) {
41
+ throw new TranscriptionFailedError(`metadata mode is not supported on model ${modelId} (engine: ${engineType || 'unknown'}); ` +
42
+ `supported engines: ${METADATA_CAPABLE_ENGINES.join(', ')}`);
27
43
  }
28
44
  }
@@ -24,6 +24,12 @@ export declare function chunkMetadata(chunk: TtsStreamChunk): {
24
24
  chunkIndex?: number;
25
25
  sampleRate?: number;
26
26
  };
27
+ /**
28
+ * Appends a chunk's samples to `target` in place, which keeps collecting an
29
+ * utterance linear in its length — a natively chunked MOSS run emits dozens of
30
+ * chunks into one buffer.
31
+ */
32
+ export declare function appendPcm(target: number[], samples: ArrayLike<number>): void;
27
33
  export declare function collectTtsStats(response: {
28
34
  stats?: AddonTtsStats;
29
35
  }): TtsStats;
@@ -14,6 +14,15 @@ export function chunkMetadata(chunk) {
14
14
  ...(chunk.isLast !== undefined ? { isLast: chunk.isLast } : {})
15
15
  };
16
16
  }
17
+ /**
18
+ * Appends a chunk's samples to `target` in place, which keeps collecting an
19
+ * utterance linear in its length — a natively chunked MOSS run emits dozens of
20
+ * chunks into one buffer.
21
+ */
22
+ export function appendPcm(target, samples) {
23
+ for (let i = 0; i < samples.length; i++)
24
+ target.push(samples[i]);
25
+ }
17
26
  // Field-for-field, apart from `audioDurationMs` -> `audioDuration`, which keeps
18
27
  // the name the SDK has always exposed.
19
28
  export function collectTtsStats(response) {
@@ -33,6 +42,10 @@ export function collectTtsStats(response) {
33
42
  ...(stats.enhancerBackendDevice !== undefined && {
34
43
  enhancerBackendDevice: stats.enhancerBackendDevice
35
44
  }),
36
- ...(stats.enhancerBackendId !== undefined && { enhancerBackendId: stats.enhancerBackendId })
45
+ ...(stats.enhancerBackendId !== undefined && { enhancerBackendId: stats.enhancerBackendId }),
46
+ ...(stats.codecSidecarLoaded !== undefined && {
47
+ codecSidecarLoaded: stats.codecSidecarLoaded
48
+ }),
49
+ ...(stats.codecOnCoreml !== undefined && { codecOnCoreml: stats.codecOnCoreml })
37
50
  };
38
51
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@qvac/inference",
3
- "version": "0.20.1",
3
+ "version": "0.21.1",
4
4
  "description": "Bare-only in-process core of the QVAC SDK. Runs inference directly on the Bare runtime with explicit plugin assembly and no RPC, worker, or subprocess layer.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -133,6 +133,14 @@
133
133
  "./ggml-classification/plugin": {
134
134
  "types": "./dist/plugins/builtin/ggml-classification/plugin.d.ts",
135
135
  "import": "./dist/plugins/builtin/ggml-classification/plugin.js"
136
+ },
137
+ "./rpc-server-provider": {
138
+ "types": "./dist/rpc/index.d.ts",
139
+ "import": "./dist/rpc/index.js"
140
+ },
141
+ "./ggml-rpc-server/provider": {
142
+ "types": "./dist/rpc/ggml-provider.d.ts",
143
+ "import": "./dist/rpc/ggml-provider.js"
136
144
  }
137
145
  },
138
146
  "files": [
@@ -158,12 +166,11 @@
158
166
  "update-models": "bare ./dist/models/update-models/index.js && prettier --write src/models/registry/models.ts src/models/registry/resource-profiles.ts"
159
167
  },
160
168
  "engines": {
161
- "bare": "^1.30.3"
169
+ "bare": ">=1.30.3"
162
170
  },
163
171
  "dependencies": {
164
172
  "@qvac/error": "^0.1.1",
165
173
  "@qvac/logging": "^0.1.1",
166
- "@qvac/model-fit": "^0.12.0",
167
174
  "@qvac/rag": "^0.8.1",
168
175
  "@qvac/registry-client": "^0.7.0",
169
176
  "bare-abort-controller": "^1.1.2",
@@ -175,10 +182,12 @@
175
182
  "bare-gpu-info": "0.1.1",
176
183
  "bare-http1": "^4.5.9",
177
184
  "bare-https": "^3.0.0",
185
+ "bare-net": "^2.3.2",
178
186
  "bare-os": "^3.9.3",
179
187
  "bare-path": "^3.1.1",
188
+ "bare-process": "^4.5.1",
180
189
  "bare-rpc": "^1.3.8",
181
- "bare-runtime": "^1.24.2",
190
+ "bare-runtime": "^1.30.3",
182
191
  "bare-stream": "^2.13.3",
183
192
  "bare-url": "^2.4.5",
184
193
  "bare-zlib": "^1.4.0",
@@ -193,19 +202,20 @@
193
202
  "zod": "^4.4.3"
194
203
  },
195
204
  "peerDependencies": {
196
- "@qvac/asr-ggml": "^0.5.3",
197
- "@qvac/audiogen-ggml": "^0.4.0",
198
- "@qvac/bci-whispercpp": "^0.9.1",
199
- "@qvac/classification-ggml": "^0.26.1",
200
- "@qvac/decoder-audio": "^0.6.0",
201
- "@qvac/diffusion-cpp": "^0.24.0",
202
- "@qvac/embed-llamacpp": "^0.41.1",
205
+ "@qvac/asr-ggml": "^0.7.1",
206
+ "@qvac/audiogen-ggml": "^0.5.2",
207
+ "@qvac/bci-whispercpp": "^0.10.2",
208
+ "@qvac/classification-ggml": "^0.28.2",
209
+ "@qvac/decoder-audio": "^0.7.0",
210
+ "@qvac/diffusion-cpp": "^0.26.1",
211
+ "@qvac/embed-llamacpp": "^0.43.2",
203
212
  "@qvac/langdetect-text": "^0.1.2",
204
- "@qvac/llm-llamacpp": "^0.53.0",
205
- "@qvac/ocr-ggml": "^0.24.1",
206
- "@qvac/translation-nmtcpp": "^0.16.1",
207
- "@qvac/tts-ggml": "^0.9.1",
208
- "@qvac/vla-ggml": "^0.27.1"
213
+ "@qvac/llm-llamacpp": "^0.55.2",
214
+ "@qvac/ocr-ggml": "^0.26.2",
215
+ "@qvac/translation-nmtcpp": "^0.18.2",
216
+ "@qvac/tts-ggml": "^0.10.2",
217
+ "@qvac/vla-ggml": "^0.29.2",
218
+ "@qvac/ggml-rpc-server": "^0.1.2"
209
219
  },
210
220
  "peerDependenciesMeta": {
211
221
  "@qvac/asr-ggml": {
@@ -246,22 +256,25 @@
246
256
  },
247
257
  "@qvac/vla-ggml": {
248
258
  "optional": true
259
+ },
260
+ "@qvac/ggml-rpc-server": {
261
+ "optional": true
249
262
  }
250
263
  },
251
264
  "devDependencies": {
252
- "@qvac/asr-ggml": "^0.5.3",
253
- "@qvac/audiogen-ggml": "^0.4.0",
254
- "@qvac/bci-whispercpp": "^0.9.1",
255
- "@qvac/classification-ggml": "^0.26.1",
256
- "@qvac/decoder-audio": "^0.6.0",
257
- "@qvac/diffusion-cpp": "^0.24.0",
258
- "@qvac/embed-llamacpp": "^0.41.1",
265
+ "@qvac/asr-ggml": "^0.7.1",
266
+ "@qvac/audiogen-ggml": "^0.5.2",
267
+ "@qvac/bci-whispercpp": "^0.10.2",
268
+ "@qvac/classification-ggml": "^0.28.2",
269
+ "@qvac/decoder-audio": "^0.7.0",
270
+ "@qvac/diffusion-cpp": "^0.26.1",
271
+ "@qvac/embed-llamacpp": "^0.43.2",
259
272
  "@qvac/langdetect-text": "^0.1.2",
260
- "@qvac/llm-llamacpp": "^0.53.0",
261
- "@qvac/ocr-ggml": "^0.24.1",
262
- "@qvac/translation-nmtcpp": "^0.16.1",
263
- "@qvac/tts-ggml": "^0.9.1",
264
- "@qvac/vla-ggml": "^0.27.1",
273
+ "@qvac/llm-llamacpp": "^0.55.2",
274
+ "@qvac/ocr-ggml": "^0.26.2",
275
+ "@qvac/translation-nmtcpp": "^0.18.2",
276
+ "@qvac/tts-ggml": "^0.10.2",
277
+ "@qvac/vla-ggml": "^0.29.2",
265
278
  "@types/brittle": "^3.5.0",
266
279
  "bare": "*",
267
280
  "bare-console": "*",
@@ -273,6 +286,7 @@
273
286
  "prettier": "^3.9.4",
274
287
  "prettier-config-holepunch": "^2.0.0",
275
288
  "tsc-alias": "^1.8.17",
276
- "typescript": "^5.9.3"
289
+ "typescript": "^5.9.3",
290
+ "@qvac/ggml-rpc-server": "^0.1.2"
277
291
  }
278
292
  }
@@ -1,2 +0,0 @@
1
- export { fetchFitStub, type FitBlobBinding, type FitStubEntry, type FitStubOptions, type FitStubOutcome, type FitStubRef, type FitStubUnavailableReason } from './fetch-fit-stub.js';
2
- export { projectFitFromStub, type StubFitInput, type StubFitOptions, type StubFitOutcome } from './project-fit-from-stub.js';
@@ -1,2 +0,0 @@
1
- export { fetchFitStub } from './fetch-fit-stub.js';
2
- export { projectFitFromStub } from './project-fit-from-stub.js';
@@ -1,31 +0,0 @@
1
- import type { CanonicalModelType } from '../../../schemas/index.js';
2
- import { type FitStubOptions, type FitStubRef, type FitStubUnavailableReason } from './fetch-fit-stub.js';
3
- import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
4
- export interface StubFitInput {
5
- model: FitStubRef;
6
- modelType: CanonicalModelType;
7
- /** The load the caller intends, in the spelling the engine's config uses. */
8
- modelConfig: unknown;
9
- }
10
- export type StubFitOutcome = {
11
- status: 'projected';
12
- fit: AdvisoryFitOutcome;
13
- } | {
14
- status: 'no-stub';
15
- reason: FitStubUnavailableReason;
16
- message?: string;
17
- };
18
- export interface StubFitOptions {
19
- stub?: FitStubOptions;
20
- fit?: AdvisoryFitOptions;
21
- }
22
- /**
23
- * Projects a load from the registry's fit stub, before the weights exist
24
- * locally.
25
- *
26
- * The stub is an ordinary GGUF as far as the fitter is concerned, so this is
27
- * the same probe `loadModel` runs, pointed at a file that is tens of KB instead
28
- * of gigabytes. It inherits that probe's fail-open contract: every failure is
29
- * an `unknown` verdict or a `no-stub` outcome, never a throw.
30
- */
31
- export declare function projectFitFromStub(input: StubFitInput, options?: StubFitOptions): Promise<StubFitOutcome>;
@@ -1,36 +0,0 @@
1
- import { fetchFitStub, removeStub } from './fetch-fit-stub.js';
2
- import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
3
- /**
4
- * Projects a load from the registry's fit stub, before the weights exist
5
- * locally.
6
- *
7
- * The stub is an ordinary GGUF as far as the fitter is concerned, so this is
8
- * the same probe `loadModel` runs, pointed at a file that is tens of KB instead
9
- * of gigabytes. It inherits that probe's fail-open contract: every failure is
10
- * an `unknown` verdict or a `no-stub` outcome, never a throw.
11
- */
12
- export async function projectFitFromStub(input, options = {}) {
13
- const stub = await fetchFitStub(input.model, options.stub);
14
- if (stub.status !== 'ready') {
15
- return stub.message === undefined
16
- ? { status: 'no-stub', reason: stub.reason }
17
- : { status: 'no-stub', reason: stub.reason, message: stub.message };
18
- }
19
- try {
20
- const fit = await runAdvisoryFitCheck({
21
- modelId: input.model.name,
22
- modelType: input.modelType,
23
- modelPath: stub.path,
24
- modelConfig: input.modelConfig,
25
- // One stub is one artifact: `fetchFitStub` does not assemble a shard set,
26
- // so nothing here may claim the fitter is looking at a split model.
27
- isShardedModel: false
28
- }, options.fit);
29
- return { status: 'projected', fit };
30
- }
31
- finally {
32
- // The stub is a per-call payload. Nothing else knows the path, so nothing
33
- // else could remove it.
34
- await removeStub(stub.path);
35
- }
36
- }