@qvac/inference 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/NOTICE +107 -33
  2. package/README.md +8 -5
  3. package/dist/api/assess-model-fit.d.ts +9 -4
  4. package/dist/api/assess-model-fit.js +9 -4
  5. package/dist/api/bci-transcribe.d.ts +4 -1
  6. package/dist/api/index.d.ts +1 -0
  7. package/dist/api/index.js +1 -0
  8. package/dist/api/rpc-server.d.ts +19 -0
  9. package/dist/api/rpc-server.js +44 -0
  10. package/dist/api/text-to-speech.d.ts +9 -0
  11. package/dist/api/text-to-speech.js +9 -0
  12. package/dist/api/transcribe.d.ts +11 -7
  13. package/dist/api/transcribe.js +47 -15
  14. package/dist/api/video-request.d.ts +5 -5
  15. package/dist/dispatch.js +39 -7
  16. package/dist/errors/index.d.ts +7 -0
  17. package/dist/errors/index.js +15 -0
  18. package/dist/handlers/assess-model-fit.d.ts +18 -6
  19. package/dist/handlers/assess-model-fit.js +215 -35
  20. package/dist/handlers/completion-orchestrate.js +17 -0
  21. package/dist/handlers/get-loaded-model-info.js +4 -1
  22. package/dist/handlers/get-model-info.js +29 -1
  23. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  24. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  25. package/dist/handlers/load-model/hyperdrive.js +2 -2
  26. package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
  27. package/dist/handlers/load-model/registry-download-utils.js +2 -1
  28. package/dist/handlers/load-model/registry.js +59 -0
  29. package/dist/handlers/rpc-server.d.ts +23 -0
  30. package/dist/handlers/rpc-server.js +141 -0
  31. package/dist/index.d.ts +2 -1
  32. package/dist/index.js +2 -1
  33. package/dist/models/registry/models.d.ts +7731 -5672
  34. package/dist/models/registry/models.js +4652 -2526
  35. package/dist/models/registry/resource-profiles.js +481 -94
  36. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  37. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  38. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  39. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  40. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  41. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  44. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
  45. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  46. package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
  47. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  48. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  49. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  50. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  51. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  52. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  54. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  55. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  56. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  57. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  58. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  59. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  60. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  61. package/dist/plugins/ops/bci-transcribe.js +18 -13
  62. package/dist/plugins/ops/load-model.js +2 -2
  63. package/dist/plugins/ops/transcribe.d.ts +4 -2
  64. package/dist/plugins/ops/transcribe.js +39 -2
  65. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  66. package/dist/plugins/ops/translate-stats.js +64 -7
  67. package/dist/plugins/ops/translate.js +20 -4
  68. package/dist/profiling/operation-metrics.js +56 -0
  69. package/dist/registry.js +4 -0
  70. package/dist/resources/model-fit/assess.d.ts +46 -2
  71. package/dist/resources/model-fit/assess.js +67 -3
  72. package/dist/resources/model-fit/estimators/llm.js +2 -2
  73. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  74. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
  75. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
  76. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  77. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  78. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  79. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  80. package/dist/resources/model-fit/floor.js +2 -2
  81. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  82. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  83. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  84. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  85. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  86. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  87. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  88. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  89. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  90. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  91. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  92. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  93. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  94. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  95. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  96. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  97. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  98. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  99. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  100. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  101. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  102. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  103. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  104. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  105. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  106. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  107. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  108. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  109. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  110. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  111. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  112. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  113. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  114. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  115. package/dist/resources/model-fit/platform.d.ts +9 -0
  116. package/dist/resources/model-fit/platform.js +28 -0
  117. package/dist/rpc/device-map.d.ts +13 -0
  118. package/dist/rpc/device-map.js +22 -0
  119. package/dist/rpc/discovery.d.ts +12 -0
  120. package/dist/rpc/discovery.js +225 -0
  121. package/dist/rpc/ggml-provider.d.ts +4 -0
  122. package/dist/rpc/ggml-provider.js +25 -0
  123. package/dist/rpc/index.d.ts +2 -0
  124. package/dist/rpc/index.js +1 -0
  125. package/dist/rpc/instance.d.ts +6 -0
  126. package/dist/rpc/instance.js +21 -0
  127. package/dist/rpc/inventory.d.ts +5 -0
  128. package/dist/rpc/inventory.js +144 -0
  129. package/dist/rpc/manager.d.ts +14 -0
  130. package/dist/rpc/manager.js +174 -0
  131. package/dist/rpc/network.d.ts +11 -0
  132. package/dist/rpc/network.js +85 -0
  133. package/dist/rpc/provider.d.ts +6 -0
  134. package/dist/rpc/provider.js +24 -0
  135. package/dist/runtime/lifecycle.js +31 -13
  136. package/dist/runtime/request-context.d.ts +1 -1
  137. package/dist/runtime/state.js +5 -0
  138. package/dist/schemas/assess-model-fit.d.ts +284 -77
  139. package/dist/schemas/assess-model-fit.js +93 -55
  140. package/dist/schemas/audio-gen.d.ts +6 -6
  141. package/dist/schemas/audio-gen.js +2 -2
  142. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  143. package/dist/schemas/batch-completion-stream.js +5 -3
  144. package/dist/schemas/bci-config.js +1 -1
  145. package/dist/schemas/bci.d.ts +86 -1
  146. package/dist/schemas/bci.js +21 -2
  147. package/dist/schemas/cancel.d.ts +10 -0
  148. package/dist/schemas/cancel.js +2 -0
  149. package/dist/schemas/common.d.ts +765 -56
  150. package/dist/schemas/common.js +7 -0
  151. package/dist/schemas/completion-event.d.ts +6 -6
  152. package/dist/schemas/completion-stream.d.ts +27 -8
  153. package/dist/schemas/completion-stream.js +41 -4
  154. package/dist/schemas/config.d.ts +236 -30
  155. package/dist/schemas/config.js +10 -0
  156. package/dist/schemas/embed.d.ts +2 -2
  157. package/dist/schemas/error.js +13 -0
  158. package/dist/schemas/errors.d.ts +1 -0
  159. package/dist/schemas/errors.js +5 -0
  160. package/dist/schemas/index.d.ts +1 -0
  161. package/dist/schemas/index.js +1 -0
  162. package/dist/schemas/llamacpp-config.d.ts +105 -14
  163. package/dist/schemas/llamacpp-config.js +113 -3
  164. package/dist/schemas/load-model.d.ts +3102 -627
  165. package/dist/schemas/load-model.js +2 -2
  166. package/dist/schemas/mcp-adapter.d.ts +4 -0
  167. package/dist/schemas/model-config-schemas.d.ts +205 -5
  168. package/dist/schemas/model-info.d.ts +56 -0
  169. package/dist/schemas/model-info.js +5 -1
  170. package/dist/schemas/plugin.d.ts +8 -2
  171. package/dist/schemas/profiling.d.ts +4 -4
  172. package/dist/schemas/reload-config.d.ts +30 -0
  173. package/dist/schemas/rpc-server.d.ts +101 -0
  174. package/dist/schemas/rpc-server.js +87 -0
  175. package/dist/schemas/sdcpp-config.d.ts +7 -3
  176. package/dist/schemas/sdcpp-config.js +37 -2
  177. package/dist/schemas/system-resources.d.ts +4 -4
  178. package/dist/schemas/text-to-speech.d.ts +529 -1
  179. package/dist/schemas/text-to-speech.js +122 -12
  180. package/dist/schemas/tools.d.ts +2 -0
  181. package/dist/schemas/tools.js +8 -0
  182. package/dist/schemas/transcription-config.d.ts +5 -0
  183. package/dist/schemas/transcription-config.js +20 -2
  184. package/dist/schemas/transcription.d.ts +164 -0
  185. package/dist/schemas/transcription.js +71 -4
  186. package/dist/schemas/vla.d.ts +1 -1
  187. package/dist/surface.d.ts +4 -1
  188. package/dist/surface.js +3 -1
  189. package/dist/utils/addon-responses.d.ts +5 -1
  190. package/dist/utils/archive.js +5 -1
  191. package/dist/utils/asr-diagnostics.d.ts +12 -0
  192. package/dist/utils/asr-diagnostics.js +42 -0
  193. package/dist/utils/asr-events.d.ts +1 -0
  194. package/dist/utils/asr-events.js +3 -2
  195. package/dist/utils/cache/paths.js +3 -2
  196. package/dist/utils/mcp-adapter.js +7 -1
  197. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  198. package/dist/utils/multi-gpu-mobile.js +8 -6
  199. package/dist/utils/path-security.d.ts +5 -0
  200. package/dist/utils/path-security.js +14 -0
  201. package/dist/utils/tool-helpers.d.ts +4 -0
  202. package/dist/utils/tool-helpers.js +2 -0
  203. package/dist/utils/tools/defer.d.ts +93 -0
  204. package/dist/utils/tools/defer.js +290 -0
  205. package/dist/utils/tools/index.d.ts +1 -0
  206. package/dist/utils/tools/index.js +1 -0
  207. package/dist/utils/transcribe-metadata.d.ts +21 -7
  208. package/dist/utils/transcribe-metadata.js +24 -8
  209. package/dist/utils/tts-stats.d.ts +6 -0
  210. package/dist/utils/tts-stats.js +14 -1
  211. package/package.json +44 -30
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  213. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
@@ -144,16 +144,47 @@ export async function createTranscribeStreamSession(params, options, process, se
144
144
  statsSettled = true;
145
145
  rejectStatsPromise(error);
146
146
  }
147
- function resolveEmptyStats() {
148
- resolveStats(undefined);
147
+ // Settled from the same terminal frame as `stats`, and on the same paths:
148
+ // resolved empty when the stream closes without one, rejected on error.
149
+ let diagnosticsSettled = false;
150
+ let resolveDiagnosticsPromise = () => { };
151
+ let rejectDiagnosticsPromise = () => { };
152
+ const diagnostics = new Promise((resolve, reject) => {
153
+ resolveDiagnosticsPromise = resolve;
154
+ rejectDiagnosticsPromise = reject;
155
+ });
156
+ diagnostics.catch(() => { });
157
+ function resolveDiagnostics(value) {
158
+ if (diagnosticsSettled)
159
+ return;
160
+ diagnosticsSettled = true;
161
+ resolveDiagnosticsPromise(value);
162
+ }
163
+ function rejectDiagnostics(error) {
164
+ if (diagnosticsSettled)
165
+ return;
166
+ diagnosticsSettled = true;
167
+ rejectDiagnosticsPromise(error);
168
+ }
169
+ function resolveTerminal(statsValue, diagnosticsValue) {
170
+ resolveStats(statsValue);
171
+ resolveDiagnostics(diagnosticsValue);
172
+ }
173
+ function resolveEmptyTerminal() {
174
+ resolveTerminal(undefined, undefined);
175
+ }
176
+ function rejectTerminal(error) {
177
+ rejectStats(error);
178
+ rejectDiagnostics(error);
149
179
  }
150
180
  function processResponse(line) {
151
- return process(line, resolveStats);
181
+ return process(line, resolveTerminal);
152
182
  }
153
- const responses = parseLines(responseStream, processResponse, resolveEmptyStats, rejectStats);
183
+ const responses = parseLines(responseStream, processResponse, resolveEmptyTerminal, rejectTerminal);
154
184
  let consumed = false;
155
185
  return {
156
186
  stats,
187
+ diagnostics,
157
188
  write(audioChunk) {
158
189
  requestStream.write(audioChunk);
159
190
  },
@@ -161,7 +192,7 @@ export async function createTranscribeStreamSession(params, options, process, se
161
192
  requestStream.end();
162
193
  },
163
194
  destroy() {
164
- resolveStats(undefined);
195
+ resolveEmptyTerminal();
165
196
  requestStream.destroy();
166
197
  responseStream.destroy();
167
198
  },
@@ -182,7 +213,7 @@ function transcribeStreamDuplexMetadata(params, options) {
182
213
  }
183
214
  function transcribeStreamDuplexConversation(params, options) {
184
215
  const wantsMetadata = params.metadata === true;
185
- return createTranscribeStreamSession(params, options, (line, onStats) => processLineConversation(line, wantsMetadata, onStats), 'TranscribeStreamConversationSession');
216
+ return createTranscribeStreamSession(params, options, (line, onTerminal) => processLineConversation(line, wantsMetadata, onTerminal), 'TranscribeStreamConversationSession');
186
217
  }
187
218
  /**
188
219
  * Line-delimited parser: reads newline-separated frames from a duplex
@@ -241,31 +272,32 @@ function parseResponseLine(line) {
241
272
  * `undefined` for frames the caller should skip, or the value extracted from
242
273
  * the frame.
243
274
  */
244
- function processWith(line, extract, onStats) {
275
+ function processWith(line, extract, onTerminal) {
245
276
  const response = parseResponseLine(line);
246
277
  if (response === null)
247
278
  return undefined;
248
279
  if (response.error)
249
280
  throw new TranscriptionFailedError(response.error);
250
281
  if (response.done) {
251
- onStats(response.stats);
282
+ onTerminal(response.stats, response.diagnostics);
252
283
  return null;
253
284
  }
254
285
  return extract(response);
255
286
  }
256
- export function processLine(line, onStats) {
257
- return processWith(line, (response) => (response.text?.trim() ? response.text : undefined), onStats);
287
+ export function processLine(line, onTerminal) {
288
+ return processWith(line, (response) => (response.text?.trim() ? response.text : undefined), onTerminal);
258
289
  }
259
- function processLineMetadata(line, onStats) {
260
- return processWith(line, (response) => response.segment, onStats);
290
+ function processLineMetadata(line, onTerminal) {
291
+ return processWith(line, (response) => response.segment, onTerminal);
261
292
  }
262
- function processLineConversation(line, wantsMetadata, onStats) {
293
+ export function processLineConversation(line, wantsMetadata, onTerminal) {
263
294
  return processWith(line, (response) => {
264
295
  if (response.vad) {
265
296
  return {
266
297
  type: 'vad',
267
298
  speaking: response.vad.speaking,
268
- probability: response.vad.probability
299
+ probability: response.vad.probability,
300
+ ...(response.vad.source && { source: response.vad.source })
269
301
  };
270
302
  }
271
303
  if (response.endOfTurn) {
@@ -291,5 +323,5 @@ function processLineConversation(line, wantsMetadata, onStats) {
291
323
  return { type: 'text', text: response.text };
292
324
  }
293
325
  return undefined;
294
- }, onStats);
326
+ }, onTerminal);
295
327
  }
@@ -6,9 +6,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
6
6
  init_image?: string;
7
7
  control_frames?: string[];
8
8
  modelId: string;
9
+ prompt: string;
9
10
  seed?: number | undefined;
10
11
  vae_tiling?: boolean | undefined;
11
- prompt: string;
12
12
  negative_prompt?: string | undefined;
13
13
  width?: number | undefined;
14
14
  height?: number | undefined;
@@ -47,9 +47,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
47
47
  init_image?: string;
48
48
  control_frames?: string[];
49
49
  modelId: string;
50
+ prompt: string;
50
51
  seed?: number | undefined;
51
52
  vae_tiling?: boolean | undefined;
52
- prompt: string;
53
53
  negative_prompt?: string | undefined;
54
54
  width?: number | undefined;
55
55
  height?: number | undefined;
@@ -88,9 +88,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
88
88
  init_image?: string;
89
89
  control_frames?: string[];
90
90
  modelId: string;
91
+ prompt: string;
91
92
  seed?: number | undefined;
92
93
  vae_tiling?: boolean | undefined;
93
- prompt: string;
94
94
  negative_prompt?: string | undefined;
95
95
  width?: number | undefined;
96
96
  height?: number | undefined;
@@ -129,9 +129,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
129
129
  init_image?: string;
130
130
  control_frames?: string[];
131
131
  modelId: string;
132
+ prompt: string;
132
133
  seed?: number | undefined;
133
134
  vae_tiling?: boolean | undefined;
134
- prompt: string;
135
135
  negative_prompt?: string | undefined;
136
136
  width?: number | undefined;
137
137
  height?: number | undefined;
@@ -170,9 +170,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
170
170
  init_image?: string;
171
171
  control_frames?: string[];
172
172
  modelId: string;
173
+ prompt: string;
173
174
  seed?: number | undefined;
174
175
  vae_tiling?: boolean | undefined;
175
- prompt: string;
176
176
  negative_prompt?: string | undefined;
177
177
  width?: number | undefined;
178
178
  height?: number | undefined;
package/dist/dispatch.js CHANGED
@@ -9,6 +9,7 @@ import { assertLifecycleAllowed } from './runtime/runtime-lifecycle.js';
9
9
  import { resolveModelConfig, setConfig, setRuntimeContext, isConfigSet } from './runtime/state.js';
10
10
  import { initialize, close as closeEngine } from './runtime/lifecycle.js';
11
11
  import { getAllPlugins } from './plugins/index.js';
12
+ import { hasRpcServerProvider } from './rpc/provider.js';
12
13
  import { resolveConfig } from './config/resolve-config.js';
13
14
  import { setGlobalLogLevel, setGlobalConsoleOutput, getAppLogger } from './logging/index.js';
14
15
  import { profileReplyHandler, profileStreamHandler } from './profiling/index.js';
@@ -24,7 +25,7 @@ let readyPromise = null;
24
25
  // does not mark readiness.
25
26
  let generation = 0;
26
27
  function ensurePluginsRegistered() {
27
- if (getAllPlugins().length === 0) {
28
+ if (getAllPlugins().length === 0 && !hasRpcServerProvider()) {
28
29
  throw new PluginsNotRegisteredError();
29
30
  }
30
31
  }
@@ -56,14 +57,15 @@ async function initializeConfig() {
56
57
  }
57
58
  async function performReady(gen) {
58
59
  initialize();
59
- ensurePluginsRegistered();
60
60
  await initializeConfig();
61
61
  // Mark ready only while this generation is current — a `close()` during the
62
62
  // await bumps it past this now-superseded init.
63
63
  if (gen === generation)
64
64
  ready = true;
65
65
  }
66
- async function ensureReady() {
66
+ async function ensureReady(allowWithoutPlugins = false) {
67
+ if (!allowWithoutPlugins)
68
+ ensurePluginsRegistered();
67
69
  if (ready)
68
70
  return;
69
71
  // A single shared promise serializes concurrent first calls: `ready` only
@@ -88,11 +90,19 @@ function getHandlerEntry(type) {
88
90
  return entry;
89
91
  }
90
92
  /**
91
- * Fill loadModel requests with device + schema config defaults before the
92
- * handler runs, matching the priority user config > device defaults > schema
93
- * defaults. Other request types pass through untouched.
93
+ * Fills a request that describes a load with device and schema config
94
+ * defaults before the handler runs, matching the priority user config >
95
+ * device defaults > schema defaults.
96
+ *
97
+ * `loadModel` and every `assessModelFit` candidate describe the same load, so
98
+ * both are filled the same way: the fitter is asked about the settings a real
99
+ * load resolves, and refuses one carrying no `device` at all. Every other
100
+ * request type passes through untouched.
94
101
  */
95
102
  function applyDeviceDefaults(request) {
103
+ if (request.type === 'assessModelFit' && 'models' in request) {
104
+ return applyCandidateDeviceDefaults(request);
105
+ }
96
106
  if (request.type !== 'loadModel' || !('modelSrc' in request))
97
107
  return request;
98
108
  let canonicalType;
@@ -105,6 +115,25 @@ function applyDeviceDefaults(request) {
105
115
  const rawConfig = request.modelConfig ?? {};
106
116
  return { ...request, modelConfig: resolveModelConfig(canonicalType, rawConfig) };
107
117
  }
118
+ function applyCandidateDeviceDefaults(request) {
119
+ const models = request.models;
120
+ if (models === undefined)
121
+ return request;
122
+ return {
123
+ ...request,
124
+ models: models.map((load) => {
125
+ let canonicalType;
126
+ try {
127
+ canonicalType = normalizeModelType(load.modelType);
128
+ }
129
+ catch {
130
+ return load;
131
+ }
132
+ const rawConfig = load.modelConfig ?? {};
133
+ return { ...load, modelConfig: resolveModelConfig(canonicalType, rawConfig) };
134
+ })
135
+ };
136
+ }
108
137
  function getProfilingMeta(request) {
109
138
  if (PROFILING_KEY in request) {
110
139
  return request[PROFILING_KEY];
@@ -178,7 +207,10 @@ async function* streamWithProgress(request, handler) {
178
207
  throw handlerError;
179
208
  }
180
209
  export async function send(request, _options) {
181
- await ensureReady();
210
+ await ensureReady(request.type === 'startRpcServer' ||
211
+ request.type === 'stopRpcServer' ||
212
+ request.type === 'discoverRpcServers' ||
213
+ (request.type === 'cancel' && request.operation === 'request'));
182
214
  assertLifecycleAllowed(request);
183
215
  const processed = prepareRequest(request);
184
216
  const entry = getHandlerEntry(processed.type);
@@ -227,6 +227,7 @@ export declare class InferenceCancelledError extends QvacErrorBase {
227
227
  readonly requestId: string;
228
228
  readonly partial: InferenceCancelledPartial;
229
229
  constructor(requestId: string, partial?: InferenceCancelledPartial, cause?: unknown);
230
+ toErrorResponseFields(): Record<string, unknown>;
230
231
  }
231
232
  export declare class AsyncDisposeUnavailableError extends QvacErrorBase {
232
233
  constructor(cause?: unknown);
@@ -433,3 +434,9 @@ export declare class PathTraversalError extends QvacErrorBase {
433
434
  export declare class ModelRegistryQueryFailedError extends QvacErrorBase {
434
435
  constructor(details?: string, cause?: unknown);
435
436
  }
437
+ export declare class RpcServerOperationError extends QvacErrorBase {
438
+ readonly operation: string;
439
+ readonly details: string;
440
+ constructor(operation: string, details: string, cause?: unknown);
441
+ toErrorResponseFields(): Record<string, unknown>;
442
+ }
@@ -354,6 +354,9 @@ export class InferenceCancelledError extends QvacErrorBase {
354
354
  this.requestId = requestId;
355
355
  this.partial = partial;
356
356
  }
357
+ toErrorResponseFields() {
358
+ return { requestId: this.requestId, partial: this.partial };
359
+ }
357
360
  }
358
361
  export class AsyncDisposeUnavailableError extends QvacErrorBase {
359
362
  constructor(cause) {
@@ -715,3 +718,15 @@ export class ModelRegistryQueryFailedError extends QvacErrorBase {
715
718
  super(createErrorOptions(ERROR_CODES.QVAC_MODEL_REGISTRY_QUERY_FAILED, details ? [details] : undefined, cause));
716
719
  }
717
720
  }
721
+ export class RpcServerOperationError extends QvacErrorBase {
722
+ operation;
723
+ details;
724
+ constructor(operation, details, cause) {
725
+ super(createErrorOptions(ERROR_CODES.RPC_SERVER_OPERATION_FAILED, [operation, details], cause));
726
+ this.operation = operation;
727
+ this.details = details;
728
+ }
729
+ toErrorResponseFields() {
730
+ return { operation: this.operation, details: this.details };
731
+ }
732
+ }
@@ -1,10 +1,22 @@
1
- import type { AssessModelFitRequest, AssessModelFitResponse } from '../schemas/assess-model-fit.js';
1
+ import type { AssessModelFitRequest, AssessModelFitResponse, ModelFitCandidate, ModelFitEstimateTarget } from '../schemas/assess-model-fit.js';
2
2
  /**
3
3
  * Runs a pre-download fit assessment worker-side.
4
4
  *
5
- * This lives on the worker because that is where the two things it needs
6
- * already are: the resource collector for a fresh memory sample, and the
7
- * runtime's own platform/arch pair. No model bytes are read and nothing is
8
- * loaded.
5
+ * This lives on the worker because that is where the three things it needs
6
+ * already are: the resource collector for a fresh memory sample, the runtime's
7
+ * own platform/arch pair, and the registry client. No weights are read and
8
+ * nothing is loaded — a single candidate additionally has the registry's
9
+ * weightless description fetched, tens of KB, so the engine's own fitter can
10
+ * answer instead of the coefficients modelling it.
9
11
  */
10
- export declare function handleAssessModelFit(request: AssessModelFitRequest): AssessModelFitResponse;
12
+ export declare function handleAssessModelFit(request: AssessModelFitRequest): Promise<AssessModelFitResponse>;
13
+ /**
14
+ * What the estimator is given for one candidate. The catalog facts come from
15
+ * the source's checksum, and the workload from the settings the load carries;
16
+ * a source outside the catalog resolves neither and assesses as `unknown`.
17
+ *
18
+ * A load whose sources are all config fields carries no checksum, so it has no
19
+ * catalog profile and the engine's own fitter is its only evidence. The model
20
+ * type stands in as the label.
21
+ */
22
+ export declare function estimateTargetFor(candidate: ModelFitCandidate): ModelFitEstimateTarget;
@@ -1,26 +1,233 @@
1
- import os from 'bare-os';
1
+ import { isCanonicalModelType, normalizeModelType, ModelType } from '../schemas/index.js';
2
+ import { inferModelTypeFromModelSrc } from '../schemas/model-src-utils.js';
3
+ import { ModelTypeRequiredError } from '../errors/index.js';
4
+ import { projectFitFromLoad } from '../resources/model-fit/fit-stub/project-fit-from-load.js';
2
5
  import { getResourceCollector } from '../resources/instance.js';
6
+ import { getConfig } from '../runtime/state.js';
3
7
  import { assessModelFitFromResources } from '../resources/model-fit/assess.js';
4
8
  import { getPlatformCalibration } from '../resources/model-fit/calibration/index.js';
9
+ import { detectPlatform } from '../resources/model-fit/platform.js';
5
10
  /**
6
11
  * Runs a pre-download fit assessment worker-side.
7
12
  *
8
- * This lives on the worker because that is where the two things it needs
9
- * already are: the resource collector for a fresh memory sample, and the
10
- * runtime's own platform/arch pair. No model bytes are read and nothing is
11
- * loaded.
13
+ * This lives on the worker because that is where the three things it needs
14
+ * already are: the resource collector for a fresh memory sample, the runtime's
15
+ * own platform/arch pair, and the registry client. No weights are read and
16
+ * nothing is loaded — a single candidate additionally has the registry's
17
+ * weightless description fetched, tens of KB, so the engine's own fitter can
18
+ * answer instead of the coefficients modelling it.
12
19
  */
13
- export function handleAssessModelFit(request) {
20
+ export async function handleAssessModelFit(request) {
14
21
  const platform = detectPlatform();
22
+ const native = await resolveNativeFit(request.models);
15
23
  const result = assessModelFitFromResources({
16
- models: request.models,
24
+ models: request.models.map(estimateTargetFor),
17
25
  execution: request.execution,
18
26
  resources: readResources(),
19
27
  platform,
20
- calibration: platform ? getPlatformCalibration(platform) : undefined
28
+ calibration: platform ? getPlatformCalibration(platform) : undefined,
29
+ nativeFit: native.fit,
30
+ nativeFitUnavailable: native.unavailable
21
31
  });
22
32
  return { type: 'assessModelFit', ...result };
23
33
  }
34
+ /**
35
+ * The engine that would run this load, named outright or read off the source,
36
+ * as `loadModel` resolves it.
37
+ *
38
+ * @throws {ModelTypeRequiredError} When neither names one.
39
+ */
40
+ function modelTypeOf(candidate) {
41
+ if (candidate.modelType !== undefined)
42
+ return normalizeModelType(candidate.modelType);
43
+ const inferred = inferModelTypeFromModelSrc(candidate.modelSrc);
44
+ if (inferred === undefined)
45
+ throw new ModelTypeRequiredError();
46
+ return normalizeModelType(inferred);
47
+ }
48
+ /** The audio engines, whose estimator sizes a load by its window rather than a context. */
49
+ const AUDIO_ENGINES = [
50
+ 'whispercpp-transcription',
51
+ 'parakeet-transcription',
52
+ 'bci-whispercpp-transcription'
53
+ ];
54
+ /**
55
+ * Engines that name a device outright, in llama's own `gpu` / `cpu` spelling.
56
+ * Both default to the GPU.
57
+ */
58
+ const DEVICE_NAMED = [ModelType.llamacppCompletion, ModelType.llamacppEmbedding];
59
+ function contextUsesGpu(config) {
60
+ const contextParams = config['contextParams'];
61
+ if (contextParams === null || typeof contextParams !== 'object')
62
+ return undefined;
63
+ return contextParams['use_gpu'];
64
+ }
65
+ /**
66
+ * Engines that carry a GPU switch, each read at the key its own config spells
67
+ * it under. These are the keys the fit builders read in `native-probe/engines`,
68
+ * so a projection and a device never disagree. Only bci defaults to on.
69
+ */
70
+ const GPU_SWITCH = {
71
+ [ModelType.ttsGgml]: (config) => config['useGPU'] === true,
72
+ [ModelType.audiogenGgml]: (config) => config['useGPU'] === true,
73
+ [ModelType.parakeetTranscription]: (config) => config['useGPU'] === true,
74
+ [ModelType.whispercppTranscription]: (config) => contextUsesGpu(config) === true,
75
+ [ModelType.bciWhispercppTranscription]: (config) => contextUsesGpu(config) !== false
76
+ };
77
+ /**
78
+ * Where a load resolved to run. `dispatch` applies the host's device defaults
79
+ * before any handler sees the config, so what arrives here is the resolved one.
80
+ *
81
+ * `nGpuLayers` wins over `useGPU` on tts, and the config schema rejects the two
82
+ * disagreeing, so reading it first is safe.
83
+ */
84
+ function resolvedDevice(modelType, config) {
85
+ if (DEVICE_NAMED.includes(modelType)) {
86
+ const device = config['device'];
87
+ return typeof device === 'string' ? device.toLowerCase() : 'gpu';
88
+ }
89
+ const layers = config['nGpuLayers'];
90
+ if (modelType === ModelType.ttsGgml && typeof layers === 'number') {
91
+ return layers === 0 ? 'cpu' : 'gpu';
92
+ }
93
+ const usesGpu = GPU_SWITCH[modelType];
94
+ if (usesGpu === undefined)
95
+ return undefined;
96
+ return usesGpu(config) ? 'gpu' : 'cpu';
97
+ }
98
+ /** The window the speech engines hold whole; longer audio is chunked into it. */
99
+ const AUDIO_WINDOW_MS = 30_000;
100
+ /**
101
+ * The audio a single call holds in memory. `duration_ms` is the longest clip
102
+ * the caller will transcribe, not a per-call window, and the engine chunks
103
+ * anything longer, so it never sizes working memory above one window.
104
+ */
105
+ function audioWindowMs(config) {
106
+ const declared = config['duration_ms'];
107
+ if (typeof declared !== 'number' || declared <= 0)
108
+ return AUDIO_WINDOW_MS;
109
+ return Math.min(declared, AUDIO_WINDOW_MS);
110
+ }
111
+ /** The catalog facts a source carries, where it carries them. */
112
+ function modelRefOf(value) {
113
+ if (value === null || typeof value !== 'object')
114
+ return undefined;
115
+ const src = value;
116
+ if (typeof src['sha256Checksum'] !== 'string')
117
+ return undefined;
118
+ const name = src['name'] ?? src['modelId'] ?? src['registryPath'];
119
+ return {
120
+ name: typeof name === 'string' ? name : '',
121
+ sha256Checksum: src['sha256Checksum'],
122
+ ...(typeof src['registryPath'] === 'string' && { registryPath: src['registryPath'] }),
123
+ ...(typeof src['registrySource'] === 'string' && { registrySource: src['registrySource'] })
124
+ };
125
+ }
126
+ /**
127
+ * The companion sources a load carries in its config — a VAD model, a pivot,
128
+ * an audiogen stage, a projector. Their bytes are part of what the set costs,
129
+ * so the estimate counts them alongside the primary.
130
+ */
131
+ function companionRefs(config) {
132
+ const refs = [];
133
+ const seen = new Set();
134
+ const add = (ref) => {
135
+ if (seen.has(ref.sha256Checksum))
136
+ return;
137
+ seen.add(ref.sha256Checksum);
138
+ refs.push(ref);
139
+ };
140
+ const walk = (key, value) => {
141
+ const ref = modelRefOf(value);
142
+ if (ref !== undefined) {
143
+ add(ref);
144
+ return;
145
+ }
146
+ if (key.endsWith('ModelSrc')) {
147
+ add({ name: typeof value === 'string' ? value : key, sha256Checksum: '' });
148
+ return;
149
+ }
150
+ if (value === null || typeof value !== 'object')
151
+ return;
152
+ for (const [inner, nested] of Object.entries(value)) {
153
+ walk(inner, nested);
154
+ }
155
+ };
156
+ for (const [key, value] of Object.entries(config))
157
+ walk(key, value);
158
+ return refs;
159
+ }
160
+ /**
161
+ * What the estimator is given for one candidate. The catalog facts come from
162
+ * the source's checksum, and the workload from the settings the load carries;
163
+ * a source outside the catalog resolves neither and assesses as `unknown`.
164
+ *
165
+ * A load whose sources are all config fields carries no checksum, so it has no
166
+ * catalog profile and the engine's own fitter is its only evidence. The model
167
+ * type stands in as the label.
168
+ */
169
+ export function estimateTargetFor(candidate) {
170
+ const descriptor = typeof candidate.modelSrc === 'string' ? undefined : candidate.modelSrc;
171
+ const location = typeof candidate.modelSrc === 'string' ? candidate.modelSrc : undefined;
172
+ const config = candidate.modelConfig ?? {};
173
+ const modelType = modelTypeOf(candidate);
174
+ const contextTokens = config['ctx_size'];
175
+ const artifacts = companionRefs(config);
176
+ const device = resolvedDevice(modelType, config);
177
+ return {
178
+ model: {
179
+ name: descriptor?.name ?? descriptor?.modelId ?? location ?? modelType,
180
+ sha256Checksum: descriptor?.sha256Checksum ?? '',
181
+ ...(descriptor?.registryPath !== undefined && { registryPath: descriptor.registryPath }),
182
+ ...(descriptor?.registrySource !== undefined && {
183
+ registrySource: descriptor.registrySource
184
+ })
185
+ },
186
+ ...(artifacts.length > 0 && { artifacts }),
187
+ ...(device !== undefined && { device }),
188
+ workload: AUDIO_ENGINES.includes(modelType)
189
+ ? {
190
+ kind: 'audio',
191
+ windowMs: audioWindowMs(config),
192
+ streaming: config['streaming'] === true
193
+ }
194
+ : {
195
+ kind: 'llm',
196
+ ...(typeof contextTokens === 'number' && contextTokens > 0 && { contextTokens })
197
+ }
198
+ };
199
+ }
200
+ /**
201
+ * The engine fitter's verdict for a single candidate, resolved through that
202
+ * load's own plugin. Only for a one-candidate request: the probe measures one
203
+ * model against the whole machine, which cannot be aggregated across a set.
204
+ */
205
+ async function resolveNativeFit(candidates) {
206
+ if (candidates.length !== 1)
207
+ return {};
208
+ const candidate = candidates[0];
209
+ if (!candidate)
210
+ return {};
211
+ const modelType = modelTypeOf(candidate);
212
+ if (!isCanonicalModelType(modelType)) {
213
+ return { unavailable: `no plugin handles model type ${modelType}` };
214
+ }
215
+ const budgetMs = getConfig().fitStubBudgetMs;
216
+ const outcome = await projectFitFromLoad({
217
+ modelType,
218
+ ...(candidate.modelSrc !== undefined && { modelSrc: candidate.modelSrc }),
219
+ ...(candidate.modelConfig !== undefined && { modelConfig: candidate.modelConfig })
220
+ }, estimateTargetFor(candidate).model.name, budgetMs === undefined ? {} : { stub: { budgetMs } });
221
+ if (outcome.status === 'projected')
222
+ return { fit: outcome.fit };
223
+ if (outcome.status === 'unsupported-load')
224
+ return { unavailable: outcome.detail };
225
+ return {
226
+ unavailable: outcome.message === undefined
227
+ ? `no registry description (${outcome.reason})`
228
+ : `no registry description (${outcome.reason}): ${outcome.message}`
229
+ };
230
+ }
24
231
  function readResources() {
25
232
  const collector = getResourceCollector();
26
233
  if (!collector) {
@@ -46,30 +253,3 @@ function readResources() {
46
253
  }
47
254
  return { capabilities: collector.getCapabilities(), sample: collector.sample() };
48
255
  }
49
- /**
50
- * Maps the runtime's platform and architecture onto a calibration target.
51
- *
52
- * @returns `undefined` for any pair this feature has no calibration target for,
53
- * which assesses as `unknown` rather than borrowing another platform's
54
- * coefficients.
55
- */
56
- function detectPlatform() {
57
- const key = `${os.platform()}-${os.arch()}`;
58
- switch (key) {
59
- case 'darwin-arm64':
60
- case 'darwin-x64':
61
- case 'linux-arm64':
62
- case 'linux-x64':
63
- case 'win32-x64':
64
- // Listed for completeness rather than reach: no engine addon is built for
65
- // win32-arm64 (`@qvac/llm-llamacpp/prebuilds` has no such target, and no
66
- // windows-arm runner exists to build one), so nothing on that platform can
67
- // load a model today. Calibrating it needs that build first, not a fixture.
68
- case 'win32-arm64':
69
- case 'android-arm64':
70
- case 'ios-arm64':
71
- return key;
72
- default:
73
- return undefined;
74
- }
75
- }
@@ -2,6 +2,7 @@ import { toolCallbackResultSchema } from '../schemas/completion-stream.js';
2
2
  import { aggregateEvents } from '../utils/aggregate-events.js';
3
3
  import { CompletionFailedError } from '../errors/index.js';
4
4
  import { dispatchPluginStream } from './plugin-dispatch.js';
5
+ import { executeToolSearch, TOOL_SEARCH_NAME } from '../utils/tools/defer.js';
5
6
  const DEFAULT_MAX_TOOL_TURNS = 8;
6
7
  // Generous ceiling on one client-side tool execution: long enough for a slow
7
8
  // MCP round trip, short enough that a client that stopped reading can't pin a
@@ -154,7 +155,23 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
154
155
  // The assistant's tool-call turn goes back verbatim so the model sees
155
156
  // its own call syntax; each result follows as a `tool` message.
156
157
  history = [...history, { role: 'assistant', content: rawFullText ?? contentText }];
158
+ // Without a deferred tool, a `tool_search` call is the caller's own tool.
159
+ const defers = request.tools?.some((tool) => tool.deferLoading === true) ?? false;
157
160
  for (const call of toolCalls) {
161
+ // `tool_search` is the SDK's own tool: it reads the registered inventory
162
+ // and appends the matched definitions, so it never leaves the worker and
163
+ // the client is never asked to run it. The `toolCall` event still went
164
+ // downstream with the turn's events, so the client can see it happened.
165
+ if (defers && call.name === TOOL_SEARCH_NAME) {
166
+ history = [
167
+ ...history,
168
+ {
169
+ role: 'tool',
170
+ content: executeToolSearch(request.tools ?? [], call.arguments, history)
171
+ }
172
+ ];
173
+ continue;
174
+ }
158
175
  yield {
159
176
  type: 'completionOrchestrate',
160
177
  turn,
@@ -15,6 +15,8 @@ export function handleGetLoadedModelInfo(request) {
15
15
  logger.warn(`getLoadedModelInfo: no plugin registered for modelType "${entry.local.modelType}" on loaded model "${modelId}"`);
16
16
  }
17
17
  const handlers = plugin ? Object.keys(plugin.handlers) : [];
18
+ const fitProbe = entry.local.fitProbe;
19
+ const probeRan = fitProbe !== undefined && fitProbe.reason !== 'disabled';
18
20
  const info = {
19
21
  modelId: entry.id,
20
22
  modelType: entry.local.modelType,
@@ -27,7 +29,8 @@ export function handleGetLoadedModelInfo(request) {
27
29
  // Same detection the completion normalizer uses, so callers see the dialect it parses.
28
30
  ...(handlers.includes('completionStream') && {
29
31
  toolDialect: detectToolDialectFromName(entry.local.name, entry.local.path)
30
- })
32
+ }),
33
+ ...(probeRan && { fitProbe })
31
34
  };
32
35
  return { type: 'getLoadedModelInfo', info };
33
36
  }