@qvac/inference 0.20.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/NOTICE +90 -24
  2. package/README.md +8 -5
  3. package/dist/api/bci-transcribe.d.ts +4 -1
  4. package/dist/api/index.d.ts +1 -0
  5. package/dist/api/index.js +1 -0
  6. package/dist/api/rpc-server.d.ts +19 -0
  7. package/dist/api/rpc-server.js +44 -0
  8. package/dist/api/text-to-speech.d.ts +9 -0
  9. package/dist/api/text-to-speech.js +9 -0
  10. package/dist/api/transcribe.d.ts +11 -7
  11. package/dist/api/transcribe.js +47 -15
  12. package/dist/api/video-request.d.ts +5 -5
  13. package/dist/dispatch.js +39 -7
  14. package/dist/errors/index.d.ts +7 -0
  15. package/dist/errors/index.js +15 -0
  16. package/dist/handlers/assess-model-fit.d.ts +8 -20
  17. package/dist/handlers/assess-model-fit.js +195 -77
  18. package/dist/handlers/completion-orchestrate.js +17 -0
  19. package/dist/handlers/get-loaded-model-info.js +4 -1
  20. package/dist/handlers/get-model-info.js +29 -1
  21. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  22. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  23. package/dist/handlers/load-model/hyperdrive.js +2 -2
  24. package/dist/handlers/load-model/registry.js +59 -0
  25. package/dist/handlers/rpc-server.d.ts +23 -0
  26. package/dist/handlers/rpc-server.js +141 -0
  27. package/dist/index.d.ts +2 -1
  28. package/dist/index.js +2 -1
  29. package/dist/models/registry/models.d.ts +7731 -5672
  30. package/dist/models/registry/models.js +4652 -2526
  31. package/dist/models/registry/resource-profiles.js +481 -94
  32. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  34. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  36. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  40. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
  41. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  42. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  43. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  44. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  45. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  46. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  47. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  48. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  49. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  50. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  51. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  52. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  53. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  54. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  55. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  56. package/dist/plugins/ops/bci-transcribe.js +18 -13
  57. package/dist/plugins/ops/load-model.js +2 -2
  58. package/dist/plugins/ops/transcribe.d.ts +4 -2
  59. package/dist/plugins/ops/transcribe.js +39 -2
  60. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  61. package/dist/plugins/ops/translate-stats.js +64 -7
  62. package/dist/plugins/ops/translate.js +20 -4
  63. package/dist/profiling/operation-metrics.js +56 -0
  64. package/dist/registry.js +4 -0
  65. package/dist/resources/model-fit/assess.d.ts +37 -2
  66. package/dist/resources/model-fit/assess.js +24 -3
  67. package/dist/resources/model-fit/estimators/llm.js +2 -2
  68. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  69. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
  70. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
  71. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  72. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  73. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  74. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  75. package/dist/resources/model-fit/floor.js +2 -2
  76. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  77. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  78. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  79. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  80. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  81. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  82. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  83. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  84. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  85. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  86. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  87. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  88. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  89. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  90. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  91. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  92. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  93. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  94. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  95. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  96. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  97. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  98. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  99. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  100. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  101. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  102. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  103. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  104. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  105. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  106. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  107. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  108. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  109. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  110. package/dist/resources/model-fit/platform.d.ts +9 -0
  111. package/dist/resources/model-fit/platform.js +28 -0
  112. package/dist/rpc/device-map.d.ts +13 -0
  113. package/dist/rpc/device-map.js +22 -0
  114. package/dist/rpc/discovery.d.ts +12 -0
  115. package/dist/rpc/discovery.js +225 -0
  116. package/dist/rpc/ggml-provider.d.ts +4 -0
  117. package/dist/rpc/ggml-provider.js +25 -0
  118. package/dist/rpc/index.d.ts +2 -0
  119. package/dist/rpc/index.js +1 -0
  120. package/dist/rpc/instance.d.ts +6 -0
  121. package/dist/rpc/instance.js +21 -0
  122. package/dist/rpc/inventory.d.ts +5 -0
  123. package/dist/rpc/inventory.js +144 -0
  124. package/dist/rpc/manager.d.ts +14 -0
  125. package/dist/rpc/manager.js +174 -0
  126. package/dist/rpc/network.d.ts +11 -0
  127. package/dist/rpc/network.js +85 -0
  128. package/dist/rpc/provider.d.ts +6 -0
  129. package/dist/rpc/provider.js +24 -0
  130. package/dist/runtime/lifecycle.js +31 -13
  131. package/dist/runtime/request-context.d.ts +1 -1
  132. package/dist/runtime/state.js +5 -0
  133. package/dist/schemas/assess-model-fit.d.ts +269 -85
  134. package/dist/schemas/assess-model-fit.js +78 -54
  135. package/dist/schemas/audio-gen.d.ts +6 -6
  136. package/dist/schemas/audio-gen.js +2 -2
  137. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  138. package/dist/schemas/batch-completion-stream.js +5 -3
  139. package/dist/schemas/bci-config.js +1 -1
  140. package/dist/schemas/bci.d.ts +86 -1
  141. package/dist/schemas/bci.js +21 -2
  142. package/dist/schemas/cancel.d.ts +10 -0
  143. package/dist/schemas/cancel.js +2 -0
  144. package/dist/schemas/common.d.ts +708 -66
  145. package/dist/schemas/common.js +7 -0
  146. package/dist/schemas/completion-event.d.ts +6 -6
  147. package/dist/schemas/completion-stream.d.ts +27 -8
  148. package/dist/schemas/completion-stream.js +41 -4
  149. package/dist/schemas/config.d.ts +104 -36
  150. package/dist/schemas/config.js +10 -0
  151. package/dist/schemas/embed.d.ts +2 -2
  152. package/dist/schemas/error.js +13 -0
  153. package/dist/schemas/errors.d.ts +1 -0
  154. package/dist/schemas/errors.js +5 -0
  155. package/dist/schemas/index.d.ts +1 -0
  156. package/dist/schemas/index.js +1 -0
  157. package/dist/schemas/llamacpp-config.d.ts +25 -18
  158. package/dist/schemas/llamacpp-config.js +26 -16
  159. package/dist/schemas/load-model.d.ts +2608 -469
  160. package/dist/schemas/load-model.js +2 -2
  161. package/dist/schemas/mcp-adapter.d.ts +4 -0
  162. package/dist/schemas/model-config-schemas.d.ts +183 -6
  163. package/dist/schemas/model-info.d.ts +56 -0
  164. package/dist/schemas/model-info.js +5 -1
  165. package/dist/schemas/plugin.d.ts +8 -2
  166. package/dist/schemas/profiling.d.ts +4 -4
  167. package/dist/schemas/reload-config.d.ts +30 -0
  168. package/dist/schemas/rpc-server.d.ts +101 -0
  169. package/dist/schemas/rpc-server.js +87 -0
  170. package/dist/schemas/sdcpp-config.d.ts +7 -3
  171. package/dist/schemas/sdcpp-config.js +37 -2
  172. package/dist/schemas/system-resources.d.ts +4 -4
  173. package/dist/schemas/text-to-speech.d.ts +529 -1
  174. package/dist/schemas/text-to-speech.js +122 -12
  175. package/dist/schemas/tools.d.ts +2 -0
  176. package/dist/schemas/tools.js +8 -0
  177. package/dist/schemas/transcription-config.d.ts +5 -0
  178. package/dist/schemas/transcription-config.js +20 -2
  179. package/dist/schemas/transcription.d.ts +164 -0
  180. package/dist/schemas/transcription.js +71 -4
  181. package/dist/schemas/vla.d.ts +1 -1
  182. package/dist/surface.d.ts +4 -1
  183. package/dist/surface.js +3 -1
  184. package/dist/utils/addon-responses.d.ts +5 -1
  185. package/dist/utils/archive.js +5 -1
  186. package/dist/utils/asr-diagnostics.d.ts +12 -0
  187. package/dist/utils/asr-diagnostics.js +42 -0
  188. package/dist/utils/asr-events.d.ts +1 -0
  189. package/dist/utils/asr-events.js +3 -2
  190. package/dist/utils/cache/paths.js +3 -2
  191. package/dist/utils/mcp-adapter.js +7 -1
  192. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  193. package/dist/utils/multi-gpu-mobile.js +8 -6
  194. package/dist/utils/path-security.d.ts +5 -0
  195. package/dist/utils/path-security.js +14 -0
  196. package/dist/utils/tool-helpers.d.ts +4 -0
  197. package/dist/utils/tool-helpers.js +2 -0
  198. package/dist/utils/tools/defer.d.ts +93 -0
  199. package/dist/utils/tools/defer.js +290 -0
  200. package/dist/utils/tools/index.d.ts +1 -0
  201. package/dist/utils/tools/index.js +1 -0
  202. package/dist/utils/transcribe-metadata.d.ts +21 -7
  203. package/dist/utils/transcribe-metadata.js +24 -8
  204. package/dist/utils/tts-stats.d.ts +6 -0
  205. package/dist/utils/tts-stats.js +14 -1
  206. package/package.json +43 -29
  207. package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
  208. package/dist/resources/model-fit/fit-stub/index.js +0 -2
  209. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
  210. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
  211. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
@@ -6,9 +6,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
6
6
  init_image?: string;
7
7
  control_frames?: string[];
8
8
  modelId: string;
9
+ prompt: string;
9
10
  seed?: number | undefined;
10
11
  vae_tiling?: boolean | undefined;
11
- prompt: string;
12
12
  negative_prompt?: string | undefined;
13
13
  width?: number | undefined;
14
14
  height?: number | undefined;
@@ -47,9 +47,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
47
47
  init_image?: string;
48
48
  control_frames?: string[];
49
49
  modelId: string;
50
+ prompt: string;
50
51
  seed?: number | undefined;
51
52
  vae_tiling?: boolean | undefined;
52
- prompt: string;
53
53
  negative_prompt?: string | undefined;
54
54
  width?: number | undefined;
55
55
  height?: number | undefined;
@@ -88,9 +88,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
88
88
  init_image?: string;
89
89
  control_frames?: string[];
90
90
  modelId: string;
91
+ prompt: string;
91
92
  seed?: number | undefined;
92
93
  vae_tiling?: boolean | undefined;
93
- prompt: string;
94
94
  negative_prompt?: string | undefined;
95
95
  width?: number | undefined;
96
96
  height?: number | undefined;
@@ -129,9 +129,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
129
129
  init_image?: string;
130
130
  control_frames?: string[];
131
131
  modelId: string;
132
+ prompt: string;
132
133
  seed?: number | undefined;
133
134
  vae_tiling?: boolean | undefined;
134
- prompt: string;
135
135
  negative_prompt?: string | undefined;
136
136
  width?: number | undefined;
137
137
  height?: number | undefined;
@@ -170,9 +170,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
170
170
  init_image?: string;
171
171
  control_frames?: string[];
172
172
  modelId: string;
173
+ prompt: string;
173
174
  seed?: number | undefined;
174
175
  vae_tiling?: boolean | undefined;
175
- prompt: string;
176
176
  negative_prompt?: string | undefined;
177
177
  width?: number | undefined;
178
178
  height?: number | undefined;
package/dist/dispatch.js CHANGED
@@ -9,6 +9,7 @@ import { assertLifecycleAllowed } from './runtime/runtime-lifecycle.js';
9
9
  import { resolveModelConfig, setConfig, setRuntimeContext, isConfigSet } from './runtime/state.js';
10
10
  import { initialize, close as closeEngine } from './runtime/lifecycle.js';
11
11
  import { getAllPlugins } from './plugins/index.js';
12
+ import { hasRpcServerProvider } from './rpc/provider.js';
12
13
  import { resolveConfig } from './config/resolve-config.js';
13
14
  import { setGlobalLogLevel, setGlobalConsoleOutput, getAppLogger } from './logging/index.js';
14
15
  import { profileReplyHandler, profileStreamHandler } from './profiling/index.js';
@@ -24,7 +25,7 @@ let readyPromise = null;
24
25
  // does not mark readiness.
25
26
  let generation = 0;
26
27
  function ensurePluginsRegistered() {
27
- if (getAllPlugins().length === 0) {
28
+ if (getAllPlugins().length === 0 && !hasRpcServerProvider()) {
28
29
  throw new PluginsNotRegisteredError();
29
30
  }
30
31
  }
@@ -56,14 +57,15 @@ async function initializeConfig() {
56
57
  }
57
58
  async function performReady(gen) {
58
59
  initialize();
59
- ensurePluginsRegistered();
60
60
  await initializeConfig();
61
61
  // Mark ready only while this generation is current — a `close()` during the
62
62
  // await bumps it past this now-superseded init.
63
63
  if (gen === generation)
64
64
  ready = true;
65
65
  }
66
- async function ensureReady() {
66
+ async function ensureReady(allowWithoutPlugins = false) {
67
+ if (!allowWithoutPlugins)
68
+ ensurePluginsRegistered();
67
69
  if (ready)
68
70
  return;
69
71
  // A single shared promise serializes concurrent first calls: `ready` only
@@ -88,11 +90,19 @@ function getHandlerEntry(type) {
88
90
  return entry;
89
91
  }
90
92
  /**
91
- * Fill loadModel requests with device + schema config defaults before the
92
- * handler runs, matching the priority user config > device defaults > schema
93
- * defaults. Other request types pass through untouched.
93
+ * Fills a request that describes a load with device and schema config
94
+ * defaults before the handler runs, matching the priority user config >
95
+ * device defaults > schema defaults.
96
+ *
97
+ * `loadModel` and every `assessModelFit` candidate describe the same load, so
98
+ * both are filled the same way: the fitter is asked about the settings a real
99
+ * load resolves, and refuses one carrying no `device` at all. Every other
100
+ * request type passes through untouched.
94
101
  */
95
102
  function applyDeviceDefaults(request) {
103
+ if (request.type === 'assessModelFit' && 'models' in request) {
104
+ return applyCandidateDeviceDefaults(request);
105
+ }
96
106
  if (request.type !== 'loadModel' || !('modelSrc' in request))
97
107
  return request;
98
108
  let canonicalType;
@@ -105,6 +115,25 @@ function applyDeviceDefaults(request) {
105
115
  const rawConfig = request.modelConfig ?? {};
106
116
  return { ...request, modelConfig: resolveModelConfig(canonicalType, rawConfig) };
107
117
  }
118
+ function applyCandidateDeviceDefaults(request) {
119
+ const models = request.models;
120
+ if (models === undefined)
121
+ return request;
122
+ return {
123
+ ...request,
124
+ models: models.map((load) => {
125
+ let canonicalType;
126
+ try {
127
+ canonicalType = normalizeModelType(load.modelType);
128
+ }
129
+ catch {
130
+ return load;
131
+ }
132
+ const rawConfig = load.modelConfig ?? {};
133
+ return { ...load, modelConfig: resolveModelConfig(canonicalType, rawConfig) };
134
+ })
135
+ };
136
+ }
108
137
  function getProfilingMeta(request) {
109
138
  if (PROFILING_KEY in request) {
110
139
  return request[PROFILING_KEY];
@@ -178,7 +207,10 @@ async function* streamWithProgress(request, handler) {
178
207
  throw handlerError;
179
208
  }
180
209
  export async function send(request, _options) {
181
- await ensureReady();
210
+ await ensureReady(request.type === 'startRpcServer' ||
211
+ request.type === 'stopRpcServer' ||
212
+ request.type === 'discoverRpcServers' ||
213
+ (request.type === 'cancel' && request.operation === 'request'));
182
214
  assertLifecycleAllowed(request);
183
215
  const processed = prepareRequest(request);
184
216
  const entry = getHandlerEntry(processed.type);
@@ -227,6 +227,7 @@ export declare class InferenceCancelledError extends QvacErrorBase {
227
227
  readonly requestId: string;
228
228
  readonly partial: InferenceCancelledPartial;
229
229
  constructor(requestId: string, partial?: InferenceCancelledPartial, cause?: unknown);
230
+ toErrorResponseFields(): Record<string, unknown>;
230
231
  }
231
232
  export declare class AsyncDisposeUnavailableError extends QvacErrorBase {
232
233
  constructor(cause?: unknown);
@@ -433,3 +434,9 @@ export declare class PathTraversalError extends QvacErrorBase {
433
434
  export declare class ModelRegistryQueryFailedError extends QvacErrorBase {
434
435
  constructor(details?: string, cause?: unknown);
435
436
  }
437
+ export declare class RpcServerOperationError extends QvacErrorBase {
438
+ readonly operation: string;
439
+ readonly details: string;
440
+ constructor(operation: string, details: string, cause?: unknown);
441
+ toErrorResponseFields(): Record<string, unknown>;
442
+ }
@@ -354,6 +354,9 @@ export class InferenceCancelledError extends QvacErrorBase {
354
354
  this.requestId = requestId;
355
355
  this.partial = partial;
356
356
  }
357
+ toErrorResponseFields() {
358
+ return { requestId: this.requestId, partial: this.partial };
359
+ }
357
360
  }
358
361
  export class AsyncDisposeUnavailableError extends QvacErrorBase {
359
362
  constructor(cause) {
@@ -715,3 +718,15 @@ export class ModelRegistryQueryFailedError extends QvacErrorBase {
715
718
  super(createErrorOptions(ERROR_CODES.QVAC_MODEL_REGISTRY_QUERY_FAILED, details ? [details] : undefined, cause));
716
719
  }
717
720
  }
721
+ export class RpcServerOperationError extends QvacErrorBase {
722
+ operation;
723
+ details;
724
+ constructor(operation, details, cause) {
725
+ super(createErrorOptions(ERROR_CODES.RPC_SERVER_OPERATION_FAILED, [operation, details], cause));
726
+ this.operation = operation;
727
+ this.details = details;
728
+ }
729
+ toErrorResponseFields() {
730
+ return { operation: this.operation, details: this.details };
731
+ }
732
+ }
@@ -1,6 +1,4 @@
1
- import type { AssessModelFitRequest, AssessModelFitResponse, ModelFitCandidate } from '../schemas/assess-model-fit.js';
2
- import { type CanonicalModelType } from '../schemas/index.js';
3
- import { type ProfileResolver } from '../resources/model-fit/assess.js';
1
+ import type { AssessModelFitRequest, AssessModelFitResponse, ModelFitCandidate, ModelFitEstimateTarget } from '../schemas/assess-model-fit.js';
4
2
  /**
5
3
  * Runs a pre-download fit assessment worker-side.
6
4
  *
@@ -12,23 +10,13 @@ import { type ProfileResolver } from '../resources/model-fit/assess.js';
12
10
  * answer instead of the coefficients modelling it.
13
11
  */
14
12
  export declare function handleAssessModelFit(request: AssessModelFitRequest): Promise<AssessModelFitResponse>;
15
- export interface FitLoad {
16
- modelType: CanonicalModelType;
17
- modelConfig: Record<string, unknown>;
18
- }
19
13
  /**
20
- * The load `loadModel` would run for this candidate, or `undefined` where no
21
- * fitter covers it.
14
+ * What the estimator is given for one candidate. The catalog facts come from
15
+ * the source's checksum, and the workload from the settings the load carries;
16
+ * a source outside the catalog resolves neither and assesses as `unknown`.
22
17
  *
23
- * The engine comes from the catalog profile, not from the workload: `llm` names
24
- * a token-context workload and covers embedding models too, which the
25
- * completion fitter refuses for asking more context than they declare. The
26
- * config carries the same device defaults a real load resolves — the fitter
27
- * answers `unsupported-config` for a load with no `device`. An embedding load
28
- * has no context setting; the fitter reads the window the model declares.
29
- *
30
- * A candidate with companion `artifacts` gets no fitter: the estimator counts
31
- * their bytes, the fitter reads one file, and a verdict for the model alone
32
- * would stand in for the set.
18
+ * A load whose sources are all config fields carries no checksum, so it has no
19
+ * catalog profile and the engine's own fitter is its only evidence. The model
20
+ * type stands in as the label.
33
21
  */
34
- export declare function fitLoad(candidate: ModelFitCandidate, resolveProfile?: ProfileResolver): FitLoad | undefined;
22
+ export declare function estimateTargetFor(candidate: ModelFitCandidate): ModelFitEstimateTarget;
@@ -1,11 +1,12 @@
1
- import os from 'bare-os';
2
- import { ModelType } from '../schemas/index.js';
3
- import { projectFitFromStub } from '../resources/model-fit/fit-stub/project-fit-from-stub.js';
4
- import { resolveModelConfig } from '../runtime/state.js';
5
- import { getModelResourceProfile } from '../models/registry/resource-profiles.js';
1
+ import { isCanonicalModelType, normalizeModelType, ModelType } from '../schemas/index.js';
2
+ import { inferModelTypeFromModelSrc } from '../schemas/model-src-utils.js';
3
+ import { ModelTypeRequiredError } from '../errors/index.js';
4
+ import { projectFitFromLoad } from '../resources/model-fit/fit-stub/project-fit-from-load.js';
6
5
  import { getResourceCollector } from '../resources/instance.js';
6
+ import { getConfig } from '../runtime/state.js';
7
7
  import { assessModelFitFromResources } from '../resources/model-fit/assess.js';
8
8
  import { getPlatformCalibration } from '../resources/model-fit/calibration/index.js';
9
+ import { detectPlatform } from '../resources/model-fit/platform.js';
9
10
  /**
10
11
  * Runs a pre-download fit assessment worker-side.
11
12
  *
@@ -18,70 +19,214 @@ import { getPlatformCalibration } from '../resources/model-fit/calibration/index
18
19
  */
19
20
  export async function handleAssessModelFit(request) {
20
21
  const platform = detectPlatform();
22
+ const native = await resolveNativeFit(request.models);
21
23
  const result = assessModelFitFromResources({
22
- models: request.models,
24
+ models: request.models.map(estimateTargetFor),
23
25
  execution: request.execution,
24
26
  resources: readResources(),
25
27
  platform,
26
28
  calibration: platform ? getPlatformCalibration(platform) : undefined,
27
- nativeFit: await resolveNativeFit(request)
29
+ nativeFit: native.fit,
30
+ nativeFitUnavailable: native.unavailable
28
31
  });
29
32
  return { type: 'assessModelFit', ...result };
30
33
  }
31
34
  /**
32
- * The engine fitter's verdict for a single candidate, read from the registry's
33
- * fit stub.
35
+ * The engine that would run this load, named outright or read off the source,
36
+ * as `loadModel` resolves it.
34
37
  *
35
- * Only for a one-candidate request: the probe measures one model against the
36
- * whole machine, which cannot be aggregated across a set. Everything else — no
37
- * stub published, an offline caller, an engine with no fit path — resolves to
38
- * `undefined` and the modelled assessment stands.
38
+ * @throws {ModelTypeRequiredError} When neither names one.
39
39
  */
40
- async function resolveNativeFit(request) {
41
- if (request.models.length !== 1)
42
- return undefined;
43
- const candidate = request.models[0];
44
- if (!candidate)
45
- return undefined;
46
- const load = fitLoad(candidate);
47
- if (!load)
40
+ function modelTypeOf(candidate) {
41
+ if (candidate.modelType !== undefined)
42
+ return normalizeModelType(candidate.modelType);
43
+ const inferred = inferModelTypeFromModelSrc(candidate.modelSrc);
44
+ if (inferred === undefined)
45
+ throw new ModelTypeRequiredError();
46
+ return normalizeModelType(inferred);
47
+ }
48
+ /** The audio engines, whose estimator sizes a load by its window rather than a context. */
49
+ const AUDIO_ENGINES = [
50
+ 'whispercpp-transcription',
51
+ 'parakeet-transcription',
52
+ 'bci-whispercpp-transcription'
53
+ ];
54
+ /**
55
+ * Engines that name a device outright, in llama's own `gpu` / `cpu` spelling.
56
+ * Both default to the GPU.
57
+ */
58
+ const DEVICE_NAMED = [ModelType.llamacppCompletion, ModelType.llamacppEmbedding];
59
+ function contextUsesGpu(config) {
60
+ const contextParams = config['contextParams'];
61
+ if (contextParams === null || typeof contextParams !== 'object')
48
62
  return undefined;
49
- const outcome = await projectFitFromStub({ model: candidate.model, ...load });
50
- return outcome.status === 'projected' ? outcome.fit : undefined;
63
+ return contextParams['use_gpu'];
51
64
  }
52
65
  /**
53
- * The load `loadModel` would run for this candidate, or `undefined` where no
54
- * fitter covers it.
55
- *
56
- * The engine comes from the catalog profile, not from the workload: `llm` names
57
- * a token-context workload and covers embedding models too, which the
58
- * completion fitter refuses for asking more context than they declare. The
59
- * config carries the same device defaults a real load resolves — the fitter
60
- * answers `unsupported-config` for a load with no `device`. An embedding load
61
- * has no context setting; the fitter reads the window the model declares.
66
+ * Engines that carry a GPU switch, each read at the key its own config spells
67
+ * it under. These are the keys the fit builders read in `native-probe/engines`,
68
+ * so a projection and a device never disagree. Only bci defaults to on.
69
+ */
70
+ const GPU_SWITCH = {
71
+ [ModelType.ttsGgml]: (config) => config['useGPU'] === true,
72
+ [ModelType.audiogenGgml]: (config) => config['useGPU'] === true,
73
+ [ModelType.parakeetTranscription]: (config) => config['useGPU'] === true,
74
+ [ModelType.whispercppTranscription]: (config) => contextUsesGpu(config) === true,
75
+ [ModelType.bciWhispercppTranscription]: (config) => contextUsesGpu(config) !== false
76
+ };
77
+ /**
78
+ * Where a load resolved to run. `dispatch` applies the host's device defaults
79
+ * before any handler sees the config, so what arrives here is the resolved one.
62
80
  *
63
- * A candidate with companion `artifacts` gets no fitter: the estimator counts
64
- * their bytes, the fitter reads one file, and a verdict for the model alone
65
- * would stand in for the set.
81
+ * `nGpuLayers` wins over `useGPU` on tts, and the config schema rejects the two
82
+ * disagreeing, so reading it first is safe.
66
83
  */
67
- export function fitLoad(candidate, resolveProfile = getModelResourceProfile) {
68
- if (candidate.workload.kind !== 'llm')
84
+ function resolvedDevice(modelType, config) {
85
+ if (DEVICE_NAMED.includes(modelType)) {
86
+ const device = config['device'];
87
+ return typeof device === 'string' ? device.toLowerCase() : 'gpu';
88
+ }
89
+ const layers = config['nGpuLayers'];
90
+ if (modelType === ModelType.ttsGgml && typeof layers === 'number') {
91
+ return layers === 0 ? 'cpu' : 'gpu';
92
+ }
93
+ const usesGpu = GPU_SWITCH[modelType];
94
+ if (usesGpu === undefined)
69
95
  return undefined;
70
- if (candidate.artifacts?.length)
96
+ return usesGpu(config) ? 'gpu' : 'cpu';
97
+ }
98
+ /** The window the speech engines hold whole; longer audio is chunked into it. */
99
+ const AUDIO_WINDOW_MS = 30_000;
100
+ /**
101
+ * The audio a single call holds in memory. `duration_ms` is the longest clip
102
+ * the caller will transcribe, not a per-call window, and the engine chunks
103
+ * anything longer, so it never sizes working memory above one window.
104
+ */
105
+ function audioWindowMs(config) {
106
+ const declared = config['duration_ms'];
107
+ if (typeof declared !== 'number' || declared <= 0)
108
+ return AUDIO_WINDOW_MS;
109
+ return Math.min(declared, AUDIO_WINDOW_MS);
110
+ }
111
+ /** The catalog facts a source carries, where it carries them. */
112
+ function modelRefOf(value) {
113
+ if (value === null || typeof value !== 'object')
71
114
  return undefined;
72
- // A model outside the catalog is taken as a completion model, the common
73
- // case for an `llm` workload.
74
- const engine = resolveProfile(candidate.model.sha256Checksum)?.engine ?? ModelType.llamacppCompletion;
75
- if (engine === ModelType.llamacppEmbedding) {
76
- return { modelType: engine, modelConfig: resolveModelConfig(engine, {}) };
77
- }
78
- if (engine === ModelType.llamacppCompletion) {
79
- return {
80
- modelType: engine,
81
- modelConfig: resolveModelConfig(engine, { ctx_size: candidate.workload.contextTokens })
82
- };
115
+ const src = value;
116
+ if (typeof src['sha256Checksum'] !== 'string')
117
+ return undefined;
118
+ const name = src['name'] ?? src['modelId'] ?? src['registryPath'];
119
+ return {
120
+ name: typeof name === 'string' ? name : '',
121
+ sha256Checksum: src['sha256Checksum'],
122
+ ...(typeof src['registryPath'] === 'string' && { registryPath: src['registryPath'] }),
123
+ ...(typeof src['registrySource'] === 'string' && { registrySource: src['registrySource'] })
124
+ };
125
+ }
126
+ /**
127
+ * The companion sources a load carries in its config — a VAD model, a pivot,
128
+ * an audiogen stage, a projector. Their bytes are part of what the set costs,
129
+ * so the estimate counts them alongside the primary.
130
+ */
131
+ function companionRefs(config) {
132
+ const refs = [];
133
+ const seen = new Set();
134
+ const add = (ref) => {
135
+ if (seen.has(ref.sha256Checksum))
136
+ return;
137
+ seen.add(ref.sha256Checksum);
138
+ refs.push(ref);
139
+ };
140
+ const walk = (key, value) => {
141
+ const ref = modelRefOf(value);
142
+ if (ref !== undefined) {
143
+ add(ref);
144
+ return;
145
+ }
146
+ if (key.endsWith('ModelSrc')) {
147
+ add({ name: typeof value === 'string' ? value : key, sha256Checksum: '' });
148
+ return;
149
+ }
150
+ if (value === null || typeof value !== 'object')
151
+ return;
152
+ for (const [inner, nested] of Object.entries(value)) {
153
+ walk(inner, nested);
154
+ }
155
+ };
156
+ for (const [key, value] of Object.entries(config))
157
+ walk(key, value);
158
+ return refs;
159
+ }
160
+ /**
161
+ * What the estimator is given for one candidate. The catalog facts come from
162
+ * the source's checksum, and the workload from the settings the load carries;
163
+ * a source outside the catalog resolves neither and assesses as `unknown`.
164
+ *
165
+ * A load whose sources are all config fields carries no checksum, so it has no
166
+ * catalog profile and the engine's own fitter is its only evidence. The model
167
+ * type stands in as the label.
168
+ */
169
+ export function estimateTargetFor(candidate) {
170
+ const descriptor = typeof candidate.modelSrc === 'string' ? undefined : candidate.modelSrc;
171
+ const location = typeof candidate.modelSrc === 'string' ? candidate.modelSrc : undefined;
172
+ const config = candidate.modelConfig ?? {};
173
+ const modelType = modelTypeOf(candidate);
174
+ const contextTokens = config['ctx_size'];
175
+ const artifacts = companionRefs(config);
176
+ const device = resolvedDevice(modelType, config);
177
+ return {
178
+ model: {
179
+ name: descriptor?.name ?? descriptor?.modelId ?? location ?? modelType,
180
+ sha256Checksum: descriptor?.sha256Checksum ?? '',
181
+ ...(descriptor?.registryPath !== undefined && { registryPath: descriptor.registryPath }),
182
+ ...(descriptor?.registrySource !== undefined && {
183
+ registrySource: descriptor.registrySource
184
+ })
185
+ },
186
+ ...(artifacts.length > 0 && { artifacts }),
187
+ ...(device !== undefined && { device }),
188
+ workload: AUDIO_ENGINES.includes(modelType)
189
+ ? {
190
+ kind: 'audio',
191
+ windowMs: audioWindowMs(config),
192
+ streaming: config['streaming'] === true
193
+ }
194
+ : {
195
+ kind: 'llm',
196
+ ...(typeof contextTokens === 'number' && contextTokens > 0 && { contextTokens })
197
+ }
198
+ };
199
+ }
200
+ /**
201
+ * The engine fitter's verdict for a single candidate, resolved through that
202
+ * load's own plugin. Only for a one-candidate request: the probe measures one
203
+ * model against the whole machine, which cannot be aggregated across a set.
204
+ */
205
+ async function resolveNativeFit(candidates) {
206
+ if (candidates.length !== 1)
207
+ return {};
208
+ const candidate = candidates[0];
209
+ if (!candidate)
210
+ return {};
211
+ const modelType = modelTypeOf(candidate);
212
+ if (!isCanonicalModelType(modelType)) {
213
+ return { unavailable: `no plugin handles model type ${modelType}` };
83
214
  }
84
- return undefined;
215
+ const budgetMs = getConfig().fitStubBudgetMs;
216
+ const outcome = await projectFitFromLoad({
217
+ modelType,
218
+ ...(candidate.modelSrc !== undefined && { modelSrc: candidate.modelSrc }),
219
+ ...(candidate.modelConfig !== undefined && { modelConfig: candidate.modelConfig })
220
+ }, estimateTargetFor(candidate).model.name, budgetMs === undefined ? {} : { stub: { budgetMs } });
221
+ if (outcome.status === 'projected')
222
+ return { fit: outcome.fit };
223
+ if (outcome.status === 'unsupported-load')
224
+ return { unavailable: outcome.detail };
225
+ return {
226
+ unavailable: outcome.message === undefined
227
+ ? `no registry description (${outcome.reason})`
228
+ : `no registry description (${outcome.reason}): ${outcome.message}`
229
+ };
85
230
  }
86
231
  function readResources() {
87
232
  const collector = getResourceCollector();
@@ -108,30 +253,3 @@ function readResources() {
108
253
  }
109
254
  return { capabilities: collector.getCapabilities(), sample: collector.sample() };
110
255
  }
111
- /**
112
- * Maps the runtime's platform and architecture onto a calibration target.
113
- *
114
- * @returns `undefined` for any pair this feature has no calibration target for,
115
- * which assesses as `unknown` rather than borrowing another platform's
116
- * coefficients.
117
- */
118
- function detectPlatform() {
119
- const key = `${os.platform()}-${os.arch()}`;
120
- switch (key) {
121
- case 'darwin-arm64':
122
- case 'darwin-x64':
123
- case 'linux-arm64':
124
- case 'linux-x64':
125
- case 'win32-x64':
126
- // Listed for completeness rather than reach: no engine addon is built for
127
- // win32-arm64 (`@qvac/llm-llamacpp/prebuilds` has no such target, and no
128
- // windows-arm runner exists to build one), so nothing on that platform can
129
- // load a model today. Calibrating it needs that build first, not a fixture.
130
- case 'win32-arm64':
131
- case 'android-arm64':
132
- case 'ios-arm64':
133
- return key;
134
- default:
135
- return undefined;
136
- }
137
- }
@@ -2,6 +2,7 @@ import { toolCallbackResultSchema } from '../schemas/completion-stream.js';
2
2
  import { aggregateEvents } from '../utils/aggregate-events.js';
3
3
  import { CompletionFailedError } from '../errors/index.js';
4
4
  import { dispatchPluginStream } from './plugin-dispatch.js';
5
+ import { executeToolSearch, TOOL_SEARCH_NAME } from '../utils/tools/defer.js';
5
6
  const DEFAULT_MAX_TOOL_TURNS = 8;
6
7
  // Generous ceiling on one client-side tool execution: long enough for a slow
7
8
  // MCP round trip, short enough that a client that stopped reading can't pin a
@@ -154,7 +155,23 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
154
155
  // The assistant's tool-call turn goes back verbatim so the model sees
155
156
  // its own call syntax; each result follows as a `tool` message.
156
157
  history = [...history, { role: 'assistant', content: rawFullText ?? contentText }];
158
+ // Without a deferred tool, a `tool_search` call is the caller's own tool.
159
+ const defers = request.tools?.some((tool) => tool.deferLoading === true) ?? false;
157
160
  for (const call of toolCalls) {
161
+ // `tool_search` is the SDK's own tool: it reads the registered inventory
162
+ // and appends the matched definitions, so it never leaves the worker and
163
+ // the client is never asked to run it. The `toolCall` event still went
164
+ // downstream with the turn's events, so the client can see it happened.
165
+ if (defers && call.name === TOOL_SEARCH_NAME) {
166
+ history = [
167
+ ...history,
168
+ {
169
+ role: 'tool',
170
+ content: executeToolSearch(request.tools ?? [], call.arguments, history)
171
+ }
172
+ ];
173
+ continue;
174
+ }
158
175
  yield {
159
176
  type: 'completionOrchestrate',
160
177
  turn,
@@ -15,6 +15,8 @@ export function handleGetLoadedModelInfo(request) {
15
15
  logger.warn(`getLoadedModelInfo: no plugin registered for modelType "${entry.local.modelType}" on loaded model "${modelId}"`);
16
16
  }
17
17
  const handlers = plugin ? Object.keys(plugin.handlers) : [];
18
+ const fitProbe = entry.local.fitProbe;
19
+ const probeRan = fitProbe !== undefined && fitProbe.reason !== 'disabled';
18
20
  const info = {
19
21
  modelId: entry.id,
20
22
  modelType: entry.local.modelType,
@@ -27,7 +29,8 @@ export function handleGetLoadedModelInfo(request) {
27
29
  // Same detection the completion normalizer uses, so callers see the dialect it parses.
28
30
  ...(handlers.includes('completionStream') && {
29
31
  toolDialect: detectToolDialectFromName(entry.local.name, entry.local.path)
30
- })
32
+ }),
33
+ ...(probeRan && { fitProbe })
31
34
  };
32
35
  return { type: 'getLoadedModelInfo', info };
33
36
  }
@@ -5,6 +5,9 @@ import { promises as fsPromises } from 'bare-fs';
5
5
  import { getShardPath, getModelsCacheDir, getSingleFileCachePath } from '../utils/cache/paths.js';
6
6
  import { validateAndJoinPath } from '../utils/path-security.js';
7
7
  import { ModelNotFoundError } from '../errors/index.js';
8
+ import { findCoremlCompanionSet, findLocallyCachedCoremlCompanionSet, getCoremlSidecarSpec, getCoremlSetKey } from './load-model/coreml-sidecars.js';
9
+ import { getRegistryClient } from '../runtime/registry-client.js';
10
+ import { getRuntimeContext } from '../runtime/state.js';
8
11
  export async function handleGetModelInfo(request) {
9
12
  const { name } = request;
10
13
  const catalogEntry = models.find((m) => m.name === name);
@@ -16,7 +19,7 @@ export async function handleGetModelInfo(request) {
16
19
  ? await handleShardedModel(cacheKey, catalogEntry.shardMetadata)
17
20
  : catalogEntry.companionSet
18
21
  ? await handleCompanionSetModel(catalogEntry.companionSet, catalogEntry.registryPath, catalogEntry.addon)
19
- : await handleSingleFileModel(catalogEntry.registryPath, catalogEntry.expectedSize, catalogEntry.sha256Checksum);
22
+ : await handleOptionalCoremlModel(catalogEntry);
20
23
  const { cacheFiles, isCached, actualSize, cachedAt, primaryPath } = cacheStatus;
21
24
  const loadedModelIds = getAllModelIds();
22
25
  const matchPath = primaryPath ?? cacheFiles[0]?.path;
@@ -64,6 +67,31 @@ export async function handleGetModelInfo(request) {
64
67
  modelInfo
65
68
  };
66
69
  }
70
+ async function handleOptionalCoremlModel(model) {
71
+ const platform = getRuntimeContext().platform;
72
+ const spec = getCoremlSidecarSpec(model.registryPath, model.registrySource, platform);
73
+ if (spec) {
74
+ const cachedSet = await findLocallyCachedCoremlCompanionSet(model, platform);
75
+ if (cachedSet) {
76
+ const cachedStatus = await handleCompanionSetModel(cachedSet, model.registryPath, model.addon);
77
+ if (cachedStatus.isCached)
78
+ return cachedStatus;
79
+ }
80
+ try {
81
+ const filename = model.registryPath.split('/').pop() || model.registryPath;
82
+ const primaryPath = validateAndJoinPath(getModelsCacheDir(), 'sets', getCoremlSetKey(model, spec), filename);
83
+ await fsPromises.access(primaryPath);
84
+ const set = await findCoremlCompanionSet(await getRegistryClient(), model, platform);
85
+ if (set) {
86
+ const status = await handleCompanionSetModel(set, model.registryPath, model.addon);
87
+ if (status.isCached)
88
+ return status;
89
+ }
90
+ }
91
+ catch { }
92
+ }
93
+ return handleSingleFileModel(model.registryPath, model.expectedSize, model.sha256Checksum);
94
+ }
67
95
  async function handleShardedModel(cacheKey, shardMetadata) {
68
96
  const fileEntries = shardMetadata.map((s) => ({
69
97
  filename: s.filename,