@qvac/inference 0.20.1 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/NOTICE +102 -40
  2. package/README.md +8 -5
  3. package/dist/api/bci-transcribe.d.ts +4 -1
  4. package/dist/api/index.d.ts +1 -0
  5. package/dist/api/index.js +1 -0
  6. package/dist/api/rpc-server.d.ts +19 -0
  7. package/dist/api/rpc-server.js +44 -0
  8. package/dist/api/text-to-speech.d.ts +9 -0
  9. package/dist/api/text-to-speech.js +9 -0
  10. package/dist/api/transcribe.d.ts +11 -7
  11. package/dist/api/transcribe.js +47 -15
  12. package/dist/api/video-request.d.ts +5 -5
  13. package/dist/dispatch.js +39 -7
  14. package/dist/errors/index.d.ts +7 -0
  15. package/dist/errors/index.js +15 -0
  16. package/dist/handlers/assess-model-fit.d.ts +8 -20
  17. package/dist/handlers/assess-model-fit.js +195 -77
  18. package/dist/handlers/completion-orchestrate.js +17 -0
  19. package/dist/handlers/get-loaded-model-info.js +4 -1
  20. package/dist/handlers/get-model-info.js +29 -1
  21. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  22. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  23. package/dist/handlers/load-model/hyperdrive.js +2 -2
  24. package/dist/handlers/load-model/registry.js +59 -0
  25. package/dist/handlers/rpc-server.d.ts +23 -0
  26. package/dist/handlers/rpc-server.js +141 -0
  27. package/dist/index.d.ts +2 -1
  28. package/dist/index.js +2 -1
  29. package/dist/models/registry/models.d.ts +7731 -5672
  30. package/dist/models/registry/models.js +4652 -2526
  31. package/dist/models/registry/resource-profiles.js +481 -94
  32. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  34. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  36. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  40. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
  41. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  42. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  43. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  44. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  45. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  46. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  47. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  48. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  49. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  50. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  51. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  52. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  53. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  54. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  55. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  56. package/dist/plugins/ops/bci-transcribe.js +18 -13
  57. package/dist/plugins/ops/load-model.js +2 -2
  58. package/dist/plugins/ops/transcribe.d.ts +4 -2
  59. package/dist/plugins/ops/transcribe.js +39 -2
  60. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  61. package/dist/plugins/ops/translate-stats.js +64 -7
  62. package/dist/plugins/ops/translate.js +20 -4
  63. package/dist/profiling/operation-metrics.js +56 -0
  64. package/dist/registry.js +4 -0
  65. package/dist/resources/model-fit/assess.d.ts +37 -2
  66. package/dist/resources/model-fit/assess.js +24 -3
  67. package/dist/resources/model-fit/estimators/llm.js +2 -2
  68. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  69. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
  70. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
  71. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  72. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  73. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  74. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  75. package/dist/resources/model-fit/floor.js +2 -2
  76. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  77. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  78. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  79. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  80. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  81. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  82. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  83. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  84. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  85. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  86. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  87. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  88. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  89. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  90. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  91. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  92. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  93. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  94. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  95. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  96. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  97. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  98. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  99. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  100. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  101. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  102. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  103. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  104. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  105. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  106. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  107. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  108. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  109. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  110. package/dist/resources/model-fit/platform.d.ts +9 -0
  111. package/dist/resources/model-fit/platform.js +28 -0
  112. package/dist/rpc/device-map.d.ts +13 -0
  113. package/dist/rpc/device-map.js +22 -0
  114. package/dist/rpc/discovery.d.ts +12 -0
  115. package/dist/rpc/discovery.js +225 -0
  116. package/dist/rpc/ggml-provider.d.ts +4 -0
  117. package/dist/rpc/ggml-provider.js +25 -0
  118. package/dist/rpc/index.d.ts +2 -0
  119. package/dist/rpc/index.js +1 -0
  120. package/dist/rpc/instance.d.ts +6 -0
  121. package/dist/rpc/instance.js +21 -0
  122. package/dist/rpc/inventory.d.ts +5 -0
  123. package/dist/rpc/inventory.js +144 -0
  124. package/dist/rpc/manager.d.ts +14 -0
  125. package/dist/rpc/manager.js +174 -0
  126. package/dist/rpc/network.d.ts +11 -0
  127. package/dist/rpc/network.js +85 -0
  128. package/dist/rpc/provider.d.ts +6 -0
  129. package/dist/rpc/provider.js +24 -0
  130. package/dist/runtime/lifecycle.js +31 -13
  131. package/dist/runtime/request-context.d.ts +1 -1
  132. package/dist/runtime/state.js +5 -0
  133. package/dist/schemas/assess-model-fit.d.ts +269 -85
  134. package/dist/schemas/assess-model-fit.js +78 -54
  135. package/dist/schemas/audio-gen.d.ts +6 -6
  136. package/dist/schemas/audio-gen.js +2 -2
  137. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  138. package/dist/schemas/batch-completion-stream.js +5 -3
  139. package/dist/schemas/bci-config.js +1 -1
  140. package/dist/schemas/bci.d.ts +86 -1
  141. package/dist/schemas/bci.js +21 -2
  142. package/dist/schemas/cancel.d.ts +10 -0
  143. package/dist/schemas/cancel.js +2 -0
  144. package/dist/schemas/common.d.ts +708 -66
  145. package/dist/schemas/common.js +7 -0
  146. package/dist/schemas/completion-event.d.ts +6 -6
  147. package/dist/schemas/completion-stream.d.ts +27 -8
  148. package/dist/schemas/completion-stream.js +41 -4
  149. package/dist/schemas/config.d.ts +104 -36
  150. package/dist/schemas/config.js +10 -0
  151. package/dist/schemas/embed.d.ts +2 -2
  152. package/dist/schemas/error.js +13 -0
  153. package/dist/schemas/errors.d.ts +1 -0
  154. package/dist/schemas/errors.js +5 -0
  155. package/dist/schemas/index.d.ts +1 -0
  156. package/dist/schemas/index.js +1 -0
  157. package/dist/schemas/llamacpp-config.d.ts +25 -18
  158. package/dist/schemas/llamacpp-config.js +26 -16
  159. package/dist/schemas/load-model.d.ts +2608 -469
  160. package/dist/schemas/load-model.js +2 -2
  161. package/dist/schemas/mcp-adapter.d.ts +4 -0
  162. package/dist/schemas/model-config-schemas.d.ts +183 -6
  163. package/dist/schemas/model-info.d.ts +56 -0
  164. package/dist/schemas/model-info.js +5 -1
  165. package/dist/schemas/plugin.d.ts +8 -2
  166. package/dist/schemas/profiling.d.ts +4 -4
  167. package/dist/schemas/reload-config.d.ts +30 -0
  168. package/dist/schemas/rpc-server.d.ts +101 -0
  169. package/dist/schemas/rpc-server.js +87 -0
  170. package/dist/schemas/sdcpp-config.d.ts +7 -3
  171. package/dist/schemas/sdcpp-config.js +37 -2
  172. package/dist/schemas/system-resources.d.ts +4 -4
  173. package/dist/schemas/text-to-speech.d.ts +529 -1
  174. package/dist/schemas/text-to-speech.js +122 -12
  175. package/dist/schemas/tools.d.ts +2 -0
  176. package/dist/schemas/tools.js +8 -0
  177. package/dist/schemas/transcription-config.d.ts +5 -0
  178. package/dist/schemas/transcription-config.js +20 -2
  179. package/dist/schemas/transcription.d.ts +164 -0
  180. package/dist/schemas/transcription.js +71 -4
  181. package/dist/schemas/vla.d.ts +1 -1
  182. package/dist/surface.d.ts +4 -1
  183. package/dist/surface.js +3 -1
  184. package/dist/utils/addon-responses.d.ts +5 -1
  185. package/dist/utils/archive.js +5 -1
  186. package/dist/utils/asr-diagnostics.d.ts +12 -0
  187. package/dist/utils/asr-diagnostics.js +42 -0
  188. package/dist/utils/asr-events.d.ts +1 -0
  189. package/dist/utils/asr-events.js +3 -2
  190. package/dist/utils/cache/paths.js +3 -2
  191. package/dist/utils/mcp-adapter.js +7 -1
  192. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  193. package/dist/utils/multi-gpu-mobile.js +8 -6
  194. package/dist/utils/path-security.d.ts +5 -0
  195. package/dist/utils/path-security.js +14 -0
  196. package/dist/utils/tool-helpers.d.ts +4 -0
  197. package/dist/utils/tool-helpers.js +2 -0
  198. package/dist/utils/tools/defer.d.ts +93 -0
  199. package/dist/utils/tools/defer.js +290 -0
  200. package/dist/utils/tools/index.d.ts +1 -0
  201. package/dist/utils/tools/index.js +1 -0
  202. package/dist/utils/transcribe-metadata.d.ts +21 -7
  203. package/dist/utils/transcribe-metadata.js +24 -8
  204. package/dist/utils/tts-stats.d.ts +6 -0
  205. package/dist/utils/tts-stats.js +14 -1
  206. package/package.json +43 -29
  207. package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
  208. package/dist/resources/model-fit/fit-stub/index.js +0 -2
  209. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
  210. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
  211. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
@@ -1,3 +1,4 @@
1
+ import { assessFit as audiogenAssessFit } from '@qvac/audiogen-ggml';
1
2
  import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
2
3
  import { resolveAudioGenConfig } from './config.js';
3
4
  export declare const audioGenPlugin: {
@@ -254,6 +255,7 @@ export declare const audioGenPlugin: {
254
255
  }>, import("zod").ZodLiteral<"vad">]>>;
255
256
  }, import("zod/v4/core").$strip>]>;
256
257
  }, import("zod/v4/core").$strict>], "engine">;
258
+ assessFit: typeof audiogenAssessFit;
257
259
  skipPrimaryModelPathValidation: true;
258
260
  resolveConfig: typeof resolveAudioGenConfig;
259
261
  createModel(params: CreateModelParams): PluginModelResult;
@@ -361,8 +363,8 @@ export declare const audioGenPlugin: {
361
363
  diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
362
364
  selectedBackend: import("zod").ZodString;
363
365
  selectedDevice: import("zod").ZodEnum<{
364
- gpu: "gpu";
365
366
  cpu: "cpu";
367
+ gpu: "gpu";
366
368
  }>;
367
369
  graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
368
370
  vulkan: "vulkan";
@@ -384,8 +386,8 @@ export declare const audioGenPlugin: {
384
386
  fallback: import("zod").ZodOptional<import("zod").ZodObject<{
385
387
  requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
386
388
  requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
387
- gpu: "gpu";
388
389
  cpu: "cpu";
390
+ gpu: "gpu";
389
391
  }>>;
390
392
  reason: import("zod").ZodString;
391
393
  }, import("zod/v4/core").$strip>>;
@@ -476,8 +478,8 @@ export declare const audioGenPlugin: {
476
478
  diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
477
479
  selectedBackend: import("zod").ZodString;
478
480
  selectedDevice: import("zod").ZodEnum<{
479
- gpu: "gpu";
480
481
  cpu: "cpu";
482
+ gpu: "gpu";
481
483
  }>;
482
484
  graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
483
485
  vulkan: "vulkan";
@@ -499,8 +501,8 @@ export declare const audioGenPlugin: {
499
501
  fallback: import("zod").ZodOptional<import("zod").ZodObject<{
500
502
  requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
501
503
  requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
502
- gpu: "gpu";
503
504
  cpu: "cpu";
505
+ gpu: "gpu";
504
506
  }>>;
505
507
  reason: import("zod").ZodString;
506
508
  }, import("zod/v4/core").$strip>>;
@@ -575,8 +577,8 @@ export declare const audioGenPlugin: {
575
577
  diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
576
578
  selectedBackend: import("zod").ZodString;
577
579
  selectedDevice: import("zod").ZodEnum<{
578
- gpu: "gpu";
579
580
  cpu: "cpu";
581
+ gpu: "gpu";
580
582
  }>;
581
583
  graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
582
584
  vulkan: "vulkan";
@@ -598,8 +600,8 @@ export declare const audioGenPlugin: {
598
600
  fallback: import("zod").ZodOptional<import("zod").ZodObject<{
599
601
  requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
600
602
  requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
601
- gpu: "gpu";
602
603
  cpu: "cpu";
604
+ gpu: "gpu";
603
605
  }>>;
604
606
  reason: import("zod").ZodString;
605
607
  }, import("zod/v4/core").$strip>>;
@@ -1,4 +1,4 @@
1
- import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX } from '@qvac/audiogen-ggml';
1
+ import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX, assessFit as audiogenAssessFit } from '@qvac/audiogen-ggml';
2
2
  import { ADDON_AUDIOGEN, ModelType, audioEditStreamRequestSchema, audioEditStreamResponseSchema, audioGenConfigSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, audioUnderstandRequestSchema, audioUnderstandResponseSchema, defineHandler, definePlugin } from '../../../schemas/index.js';
3
3
  import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
4
4
  import { resolveAudioGenConfig } from './config.js';
@@ -11,6 +11,7 @@ export const audioGenPlugin = definePlugin({
11
11
  displayName: 'Audio Generation (GGML / ACE-Step and MiniMax)',
12
12
  addonPackage: ADDON_AUDIOGEN,
13
13
  loadConfigSchema: audioGenConfigSchema,
14
+ assessFit: audiogenAssessFit,
14
15
  // AudioGen's primary `modelSrc` is intentionally empty: all required
15
16
  // weights are config-owned artifacts (ACE-Step: four sources; MiniMax: LM + synth).
16
17
  skipPrimaryModelPathValidation: true,
@@ -1,3 +1,4 @@
1
+ import { assessFit as bciAssessFit } from '@qvac/bci-whispercpp';
1
2
  import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
2
3
  import { resolveBciConfig } from './resolve-config.js';
3
4
  export declare const bciPlugin: {
@@ -76,6 +77,7 @@ export declare const bciPlugin: {
76
77
  }>, import("zod").ZodLiteral<"vad">]>>;
77
78
  }, import("zod/v4/core").$strip>]>>;
78
79
  }, import("zod/v4/core").$strip>;
80
+ assessFit: typeof bciAssessFit;
79
81
  resolveConfig: typeof resolveBciConfig;
80
82
  createModel(params: CreateModelParams): PluginModelResult;
81
83
  handlers: {
@@ -110,6 +112,18 @@ export declare const bciPlugin: {
110
112
  gpuUnsupported: import("zod").ZodOptional<import("zod").ZodNumber>;
111
113
  gpuMemTotalMb: import("zod").ZodOptional<import("zod").ZodNumber>;
112
114
  gpuMemFreeMb: import("zod").ZodOptional<import("zod").ZodNumber>;
115
+ totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
116
+ totalWallMs: import("zod").ZodOptional<import("zod").ZodNumber>;
117
+ totalSamples: import("zod").ZodOptional<import("zod").ZodNumber>;
118
+ processCalls: import("zod").ZodOptional<import("zod").ZodNumber>;
119
+ whisperSampleMs: import("zod").ZodOptional<import("zod").ZodNumber>;
120
+ whisperBatchdMs: import("zod").ZodOptional<import("zod").ZodNumber>;
121
+ whisperPromptMs: import("zod").ZodOptional<import("zod").ZodNumber>;
122
+ totalTranscriptions: import("zod").ZodOptional<import("zod").ZodNumber>;
123
+ modelLoadMs: import("zod").ZodOptional<import("zod").ZodNumber>;
124
+ totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
125
+ encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
126
+ encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
113
127
  }, import("zod/v4/core").$strip>>;
114
128
  error: import("zod").ZodOptional<import("zod").ZodString>;
115
129
  segment: import("zod").ZodOptional<import("zod").ZodObject<{
@@ -118,8 +132,51 @@ export declare const bciPlugin: {
118
132
  endMs: import("zod").ZodNumber;
119
133
  append: import("zod").ZodBoolean;
120
134
  id: import("zod").ZodNumber;
135
+ isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
136
+ startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
121
137
  }, import("zod/v4/core").$strip>>;
122
138
  type: import("zod").ZodLiteral<"bciTranscribe">;
139
+ diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
140
+ selectedBackend: import("zod").ZodString;
141
+ selectedDevice: import("zod").ZodEnum<{
142
+ cpu: "cpu";
143
+ gpu: "gpu";
144
+ }>;
145
+ graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
146
+ vulkan: "vulkan";
147
+ metal: "metal";
148
+ opencl: "opencl";
149
+ cuda: "cuda";
150
+ opengl: "opengl";
151
+ webgpu: "webgpu";
152
+ direct3d11: "direct3d11";
153
+ direct3d12: "direct3d12";
154
+ levelZero: "levelZero";
155
+ rocm: "rocm";
156
+ }>>;
157
+ driver: import("zod").ZodOptional<import("zod").ZodObject<{
158
+ name: import("zod").ZodString;
159
+ version: import("zod").ZodOptional<import("zod").ZodString>;
160
+ }, import("zod/v4/core").$strip>>;
161
+ gpuId: import("zod").ZodOptional<import("zod").ZodString>;
162
+ fallback: import("zod").ZodOptional<import("zod").ZodObject<{
163
+ requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
164
+ requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
165
+ cpu: "cpu";
166
+ gpu: "gpu";
167
+ }>>;
168
+ reason: import("zod").ZodString;
169
+ }, import("zod/v4/core").$strip>>;
170
+ probe: import("zod").ZodOptional<import("zod").ZodObject<{
171
+ status: import("zod").ZodEnum<{
172
+ unknown: "unknown";
173
+ compatible: "compatible";
174
+ incompatible: "incompatible";
175
+ }>;
176
+ backend: import("zod").ZodString;
177
+ reason: import("zod").ZodOptional<import("zod").ZodString>;
178
+ }, import("zod/v4/core").$strip>>;
179
+ }, import("zod/v4/core").$strip>>;
123
180
  }, import("zod/v4/core").$strip>>;
124
181
  bciTranscribeStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
125
182
  modelId: import("zod").ZodString;
@@ -153,16 +210,31 @@ export declare const bciPlugin: {
153
210
  gpuUnsupported: import("zod").ZodOptional<import("zod").ZodNumber>;
154
211
  gpuMemTotalMb: import("zod").ZodOptional<import("zod").ZodNumber>;
155
212
  gpuMemFreeMb: import("zod").ZodOptional<import("zod").ZodNumber>;
213
+ totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
214
+ totalWallMs: import("zod").ZodOptional<import("zod").ZodNumber>;
215
+ totalSamples: import("zod").ZodOptional<import("zod").ZodNumber>;
216
+ processCalls: import("zod").ZodOptional<import("zod").ZodNumber>;
217
+ whisperSampleMs: import("zod").ZodOptional<import("zod").ZodNumber>;
218
+ whisperBatchdMs: import("zod").ZodOptional<import("zod").ZodNumber>;
219
+ whisperPromptMs: import("zod").ZodOptional<import("zod").ZodNumber>;
220
+ totalTranscriptions: import("zod").ZodOptional<import("zod").ZodNumber>;
221
+ modelLoadMs: import("zod").ZodOptional<import("zod").ZodNumber>;
222
+ totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
223
+ encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
224
+ encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
156
225
  }, import("zod/v4/core").$strip>>;
157
226
  error: import("zod").ZodOptional<import("zod").ZodString>;
227
+ type: import("zod").ZodLiteral<"bciTranscribeStream">;
158
228
  segment: import("zod").ZodOptional<import("zod").ZodObject<{
159
229
  text: import("zod").ZodString;
160
230
  startMs: import("zod").ZodNumber;
161
231
  endMs: import("zod").ZodNumber;
162
232
  append: import("zod").ZodBoolean;
163
233
  id: import("zod").ZodNumber;
234
+ isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
235
+ startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
236
+ windowStartTimestep: import("zod").ZodOptional<import("zod").ZodNumber>;
164
237
  }, import("zod/v4/core").$strip>>;
165
- type: import("zod").ZodLiteral<"bciTranscribeStream">;
166
238
  }, import("zod/v4/core").$strip>>;
167
239
  };
168
240
  logging: {
@@ -1,8 +1,9 @@
1
- import BCIWhispercpp from '@qvac/bci-whispercpp';
1
+ import BCIWhispercpp, { assessFit as bciAssessFit } from '@qvac/bci-whispercpp';
2
2
  import { definePlugin, defineHandler, defineDuplexHandler, bciTranscribeRequestSchema, bciTranscribeResponseSchema, bciTranscribeStreamRequestSchema, bciTranscribeStreamResponseSchema, ModelType, bciConfigSchema, ADDON_BCI } from '../../../schemas/index.js';
3
3
  import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
4
4
  import { bciTranscribe, bciTranscribeStream } from '../../ops/bci-transcribe.js';
5
5
  import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
6
+ import { attachBackendDiagnostics } from '../../../profiling/backend-diagnostics.js';
6
7
  import { buildBciWhispercppArgs } from './args.js';
7
8
  import { resolveBciConfig } from './resolve-config.js';
8
9
  function createBciModel(modelId, modelPath, bciConfig, embedderPath) {
@@ -17,6 +18,7 @@ export const bciPlugin = definePlugin({
17
18
  displayName: 'BCI (whisper.cpp)',
18
19
  addonPackage: ADDON_BCI,
19
20
  loadConfigSchema: bciConfigSchema,
21
+ assessFit: bciAssessFit,
20
22
  resolveConfig: resolveBciConfig,
21
23
  createModel(params) {
22
24
  const bciConfig = (params.modelConfig ?? {});
@@ -57,13 +59,17 @@ export const bciPlugin = definePlugin({
57
59
  };
58
60
  result = await stream.next();
59
61
  }
60
- const { modelExecutionMs, stats } = result.value;
61
- yield attachModelExecutionMs({
62
+ const { modelExecutionMs, stats, diagnostics } = result.value;
63
+ // The field is what reaches an RPC client; the symbol is what the
64
+ // profiling layer reads to set `event.backend`, as audiogen does.
65
+ const terminal = attachModelExecutionMs({
62
66
  type: 'bciTranscribe',
63
67
  text: '',
64
68
  done: true,
65
- ...(stats && { stats })
69
+ ...(stats && { stats }),
70
+ ...(diagnostics && { diagnostics })
66
71
  }, modelExecutionMs);
72
+ yield diagnostics ? attachBackendDiagnostics(terminal, diagnostics) : terminal;
67
73
  }
68
74
  finally {
69
75
  await stream.return?.(undefined);
@@ -84,22 +90,34 @@ export const bciPlugin = definePlugin({
84
90
  const iterator = metadata
85
91
  ? bciTranscribeStream(request.modelId, inputStream, true, request.streamOpts, request.requestId)
86
92
  : bciTranscribeStream(request.modelId, inputStream, false, request.streamOpts, request.requestId);
87
- for await (const value of iterator) {
88
- yield metadata
89
- ? {
90
- type: 'bciTranscribeStream',
91
- segment: value
92
- }
93
- : {
94
- type: 'bciTranscribeStream',
95
- text: value
96
- };
93
+ // Iterated by hand rather than with `for await` so the generator's
94
+ // return value survives: it carries the execution time the profiling
95
+ // layer reads off the terminal frame.
96
+ try {
97
+ let result = await iterator.next();
98
+ while (!result.done) {
99
+ yield metadata
100
+ ? {
101
+ type: 'bciTranscribeStream',
102
+ segment: result.value
103
+ }
104
+ : {
105
+ type: 'bciTranscribeStream',
106
+ text: result.value
107
+ };
108
+ result = await iterator.next();
109
+ }
110
+ // The addon reports no stats for a stream, so there is no backend
111
+ // verdict to attach here — only the timing.
112
+ yield attachModelExecutionMs({
113
+ type: 'bciTranscribeStream',
114
+ text: '',
115
+ done: true
116
+ }, result.value.modelExecutionMs);
117
+ }
118
+ finally {
119
+ await iterator.return?.(undefined);
97
120
  }
98
- yield {
99
- type: 'bciTranscribeStream',
100
- text: '',
101
- done: true
102
- };
103
121
  }
104
122
  })
105
123
  },
@@ -5,8 +5,8 @@ export declare const vlaPlugin: {
5
5
  addonPackage: "@qvac/vla-ggml";
6
6
  loadConfigSchema: import("zod").ZodObject<{
7
7
  backend: import("zod").ZodOptional<import("zod").ZodEnum<{
8
- auto: "auto";
9
8
  cpu: "cpu";
9
+ auto: "auto";
10
10
  }>>;
11
11
  verbosity: import("zod").ZodOptional<import("zod").ZodNumber>;
12
12
  embodiment: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodNumber, import("zod").ZodObject<{
@@ -6,6 +6,7 @@ import { getResponseFormatJsonSchema } from '../../../../utils/response-format.j
6
6
  import { seedConfiguredSystemPrompt, transformMessages } from './completion-stream.js';
7
7
  import { normalizeCompletionStats } from './completion-stats.js';
8
8
  import { prependToolsToHistory } from '../../../../utils/tool-integration.js';
9
+ import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
9
10
  const logger = getEngineLogger();
10
11
  function runBatchModel(model, prompts) {
11
12
  const run = model.run.bind(model);
@@ -22,9 +23,10 @@ function mergeGenerationParams(generationParams, responseFormat) {
22
23
  json_schema: jsonSchema
23
24
  };
24
25
  }
25
- function renderPromptHistory(prompt, options) {
26
- const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0 ? prompt.tools : undefined;
27
- const history = seedConfiguredSystemPrompt(prompt.history, options.modelConfig);
26
+ function renderPromptHistory(prompt, history, deferred, options) {
27
+ const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0
28
+ ? (deferred?.toolsToRender ?? prompt.tools.map(toWireTool))
29
+ : undefined;
28
30
  let historyWithTools = history;
29
31
  if (tools) {
30
32
  historyWithTools = prependToolsToHistory(history, tools);
@@ -34,10 +36,12 @@ function renderPromptHistory(prompt, options) {
34
36
  return transformMessages(historyWithTools);
35
37
  }
36
38
  function buildBatchPrompt(prompt, options) {
37
- const mergedGenerationParams = mergeGenerationParams(prompt.generationParams, prompt.responseFormat);
39
+ const history = seedConfiguredSystemPrompt(prompt.history, options.modelConfig);
40
+ const deferred = resolveDeferredTools(prompt.tools, history);
41
+ const mergedGenerationParams = withDeferredToolChoice(mergeGenerationParams(prompt.generationParams, prompt.responseFormat), options.toolsEnabled ? deferred : null);
38
42
  return {
39
43
  ...(prompt.id !== undefined && { id: prompt.id }),
40
- prompt: renderPromptHistory(prompt, options),
44
+ prompt: renderPromptHistory(prompt, history, deferred, options),
41
45
  ...(mergedGenerationParams && {
42
46
  runOptions: { generationParams: mergedGenerationParams }
43
47
  })
@@ -10,7 +10,7 @@ export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
10
10
  emittedTokens?: number | undefined;
11
11
  avgConcurrentSeq?: number | undefined;
12
12
  toolDefinitionsDropped?: number | undefined;
13
- backendDevice?: "gpu" | "cpu" | undefined;
13
+ backendDevice?: "cpu" | "gpu" | undefined;
14
14
  } | undefined;
15
15
  /**
16
16
  * Attach the count of non-empty pieces the addon streamed without
@@ -4,6 +4,7 @@ import { getModel, getModelConfig } from '../../../../runtime/model-registry.js'
4
4
  import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
5
5
  import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
6
6
  import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
7
+ import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
7
8
  import { parseToolCalls } from '../../../../utils/tools/index.js';
8
9
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
9
10
  import { toolChoiceDemandsCall } from '../../../../schemas/completion-stream.js';
@@ -209,7 +210,14 @@ export async function* completion(params, opts) {
209
210
  const modelConfig = getModelConfig(modelId);
210
211
  const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
211
212
  const toolsEnabled = modelConfig.tools === true;
212
- const toolsActive = !!tools?.length && toolsEnabled;
213
+ // With deferred tools the prompt and the parser see different lists: the
214
+ // prompt carries the always-loaded ones plus `tool_search` and its catalog,
215
+ // while the parser also accepts whatever earlier searches appended to the
216
+ // history. `null` when nothing defers, which leaves the existing path alone.
217
+ const deferred = resolveDeferredTools(tools, history);
218
+ const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
219
+ const callableTools = deferred?.callableTools ?? toolsToRender;
220
+ const toolsActive = !!toolsToRender?.length && toolsEnabled;
213
221
  const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
214
222
  // `responseFormat` is forwarded to the addon as a per-request
215
223
  // `generationParams.json_schema`, which the addon converts to GBNF and
@@ -228,6 +236,7 @@ export async function* completion(params, opts) {
228
236
  };
229
237
  }
230
238
  }
239
+ mergedGenerationParams = withDeferredToolChoice(mergedGenerationParams, toolsActive ? deferred : null);
231
240
  const model = getModel(modelId);
232
241
  // Per-request hard cancel: under continuous batching the model runs several
233
242
  // jobs at once, so an abort must cancel only THIS request's job, not the
@@ -261,13 +270,13 @@ export async function* completion(params, opts) {
261
270
  if (!kvCache) {
262
271
  // KV-cache disabled — straight passthrough, no session involvement.
263
272
  let historyWithTools = history;
264
- if (toolsActive && tools) {
265
- historyWithTools = prependToolsToHistory(history, tools);
273
+ if (toolsActive && toolsToRender) {
274
+ historyWithTools = prependToolsToHistory(history, toolsToRender);
266
275
  }
267
276
  const transformedHistory = transformMessages(historyWithTools);
268
277
  logCacheDisabled();
269
278
  logMessagesToAddon(transformedHistory, 'NO_CACHE');
270
- return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect, setActiveResponse);
279
+ return yield* processModelResponse(model, transformedHistory, callableTools, mergedGenerationParams, undefined, dialect, setActiveResponse);
271
280
  }
272
281
  // ---- KV-cache path. The session owns every bookkeeping layer; the handler
273
282
  // registers one deferred unwind that `commitTurn` short-circuits on the happy
@@ -281,7 +290,7 @@ export async function* completion(params, opts) {
281
290
  // The tool block is baked into the cache on the turn that first sends it and
282
291
  // never trimmed, so a late or changed tool set has to land on a fresh cache
283
292
  // rather than a warm prefix holding the old block.
284
- const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? tools : undefined);
293
+ const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? toolsToRender : undefined);
285
294
  let turn;
286
295
  if (typeof kvCache === 'string') {
287
296
  turn = await session.beginTurn({
@@ -315,7 +324,7 @@ export async function* completion(params, opts) {
315
324
  scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
316
325
  let payload;
317
326
  try {
318
- payload = await prepareMessagesForCache(session, turn, history, toolsActive ? tools : undefined, mergedGenerationParams?.tool_choice);
327
+ payload = await prepareMessagesForCache(session, turn, history, toolsActive ? toolsToRender : undefined, mergedGenerationParams?.tool_choice);
319
328
  }
320
329
  catch (error) {
321
330
  // A missing attachment is caller input rejected before the addon runs,
@@ -328,7 +337,7 @@ export async function* completion(params, opts) {
328
337
  let result;
329
338
  let addonRunSettled = false;
330
339
  try {
331
- result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
340
+ result = yield* processModelResponse(model, messagesToSend, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
332
341
  addonRunSettled = true;
333
342
  });
334
343
  }
@@ -39,15 +39,17 @@ export declare const llmPlugin: {
39
39
  integrated: "integrated";
40
40
  dedicated: "dedicated";
41
41
  }>]>>;
42
+ 'rpc-servers': import("zod").ZodOptional<import("zod").ZodString>;
43
+ devices: import("zod").ZodOptional<import("zod").ZodString>;
42
44
  'split-mode': import("zod").ZodOptional<import("zod").ZodEnum<{
43
45
  none: "none";
44
46
  layer: "layer";
45
47
  tensor: "tensor";
46
48
  }>>;
47
49
  'flash-attn': import("zod").ZodOptional<import("zod").ZodEnum<{
50
+ auto: "auto";
48
51
  on: "on";
49
52
  off: "off";
50
- auto: "auto";
51
53
  }>>;
52
54
  'tensor-split': import("zod").ZodOptional<import("zod").ZodString>;
53
55
  'batch-size': import("zod").ZodOptional<import("zod").ZodNumber>;
@@ -66,9 +68,9 @@ export declare const llmPlugin: {
66
68
  'moe-cache-mib': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodLiteral<"auto">]>>;
67
69
  'prefetch-weights': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodBoolean, import("zod").ZodLiteral<"auto">]>>;
68
70
  'tensor-read-lazy': import("zod").ZodOptional<import("zod").ZodEnum<{
71
+ auto: "auto";
69
72
  on: "on";
70
73
  off: "off";
71
- auto: "auto";
72
74
  }>>;
73
75
  fit: import("zod").ZodOptional<import("zod").ZodBoolean>;
74
76
  'fit-target': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodString]>>;
@@ -125,6 +127,7 @@ export declare const llmPlugin: {
125
127
  }>>;
126
128
  'mmproj-use-gpu': import("zod").ZodOptional<import("zod").ZodBoolean>;
127
129
  }, import("zod/v4/core").$strip>;
130
+ assessFit: typeof import("node_modules/@qvac/llm-llamacpp/fit").assessFit;
128
131
  resolveConfig(cfg: LlmConfigInput, ctx: ResolveContext): Promise<{
129
132
  config: {
130
133
  ctx_size?: number | undefined;
@@ -148,8 +151,10 @@ export declare const llmPlugin: {
148
151
  'cache-type-k'?: string | undefined;
149
152
  'cache-type-v'?: string | undefined;
150
153
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
154
+ 'rpc-servers'?: string | undefined;
155
+ devices?: string | undefined;
151
156
  'split-mode'?: "none" | "layer" | "tensor" | undefined;
152
- 'flash-attn'?: "on" | "off" | "auto" | undefined;
157
+ 'flash-attn'?: "auto" | "on" | "off" | undefined;
153
158
  'tensor-split'?: string | undefined;
154
159
  'batch-size'?: number | undefined;
155
160
  'ubatch-size'?: number | undefined;
@@ -166,7 +171,7 @@ export declare const llmPlugin: {
166
171
  'n-cpu-ffn'?: number | undefined;
167
172
  'moe-cache-mib'?: number | "auto" | undefined;
168
173
  'prefetch-weights'?: boolean | "auto" | undefined;
169
- 'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
174
+ 'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
170
175
  fit?: boolean | undefined;
171
176
  'fit-target'?: string | number | undefined;
172
177
  'fit-ctx'?: number | undefined;
@@ -200,8 +205,10 @@ export declare const llmPlugin: {
200
205
  'cache-type-k'?: string | undefined;
201
206
  'cache-type-v'?: string | undefined;
202
207
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
208
+ 'rpc-servers'?: string | undefined;
209
+ devices?: string | undefined;
203
210
  'split-mode'?: "none" | "layer" | "tensor" | undefined;
204
- 'flash-attn'?: "on" | "off" | "auto" | undefined;
211
+ 'flash-attn'?: "auto" | "on" | "off" | undefined;
205
212
  'tensor-split'?: string | undefined;
206
213
  'batch-size'?: number | undefined;
207
214
  'ubatch-size'?: number | undefined;
@@ -218,7 +225,7 @@ export declare const llmPlugin: {
218
225
  'n-cpu-ffn'?: number | undefined;
219
226
  'moe-cache-mib'?: number | "auto" | undefined;
220
227
  'prefetch-weights'?: boolean | "auto" | undefined;
221
- 'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
228
+ 'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
222
229
  fit?: boolean | undefined;
223
230
  'fit-target'?: string | number | undefined;
224
231
  'fit-ctx'?: number | undefined;
@@ -288,6 +295,8 @@ export declare const llmPlugin: {
288
295
  type: import("zod").ZodLiteral<"function">;
289
296
  name: import("zod").ZodString;
290
297
  description: import("zod").ZodString;
298
+ deferLoading: import("zod").ZodOptional<import("zod").ZodBoolean>;
299
+ group: import("zod").ZodOptional<import("zod").ZodString>;
291
300
  parameters: import("zod").ZodObject<{
292
301
  type: import("zod").ZodLiteral<"object">;
293
302
  properties: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodObject<{
@@ -360,8 +369,8 @@ export declare const llmPlugin: {
360
369
  avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
361
370
  toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
362
371
  backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
363
- gpu: "gpu";
364
372
  cpu: "cpu";
373
+ gpu: "gpu";
365
374
  }>>;
366
375
  }, import("zod/v4/core").$strip>;
367
376
  }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
@@ -379,8 +388,8 @@ export declare const llmPlugin: {
379
388
  seq: import("zod").ZodNumber;
380
389
  stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
381
390
  cancelled: "cancelled";
382
- eos: "eos";
383
391
  length: "length";
392
+ eos: "eos";
384
393
  stopSequence: "stopSequence";
385
394
  }>>;
386
395
  raw: import("zod").ZodOptional<import("zod").ZodObject<{
@@ -399,8 +408,8 @@ export declare const llmPlugin: {
399
408
  avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
400
409
  toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
401
410
  backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
402
- gpu: "gpu";
403
411
  cpu: "cpu";
412
+ gpu: "gpu";
404
413
  }>>;
405
414
  }, import("zod/v4/core").$strip>>;
406
415
  }, import("zod/v4/core").$strict>>;
@@ -417,6 +426,8 @@ export declare const llmPlugin: {
417
426
  type: import("zod").ZodLiteral<"function">;
418
427
  name: import("zod").ZodString;
419
428
  description: import("zod").ZodString;
429
+ deferLoading: import("zod").ZodOptional<import("zod").ZodBoolean>;
430
+ group: import("zod").ZodOptional<import("zod").ZodString>;
420
431
  parameters: import("zod").ZodObject<{
421
432
  type: import("zod").ZodLiteral<"object">;
422
433
  properties: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodObject<{
@@ -525,8 +536,8 @@ export declare const llmPlugin: {
525
536
  avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
526
537
  toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
527
538
  backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
528
- gpu: "gpu";
529
539
  cpu: "cpu";
540
+ gpu: "gpu";
530
541
  }>>;
531
542
  }, import("zod/v4/core").$strip>;
532
543
  }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
@@ -544,8 +555,8 @@ export declare const llmPlugin: {
544
555
  seq: import("zod").ZodNumber;
545
556
  stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
546
557
  cancelled: "cancelled";
547
- eos: "eos";
548
558
  length: "length";
559
+ eos: "eos";
549
560
  stopSequence: "stopSequence";
550
561
  }>>;
551
562
  raw: import("zod").ZodOptional<import("zod").ZodObject<{
@@ -71,14 +71,15 @@ import { isAddonContextOverflowError, parseContextOverflowMessage } from './ops/
71
71
  import { stoppedByLength } from './ops/completion-stats.js';
72
72
  import { isMobile } from '../../../runtime/state.js';
73
73
  import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
74
+ import { resolveDeferredTools, toWireTool } from '../../../utils/tools/defer.js';
74
75
  function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
75
76
  const logger = createStreamLogger(modelId, ModelType.llamacppCompletion);
76
77
  registerAddonLogger(modelId, ModelType.llamacppCompletion, logger);
77
78
  const llmConfigStrings = transformLlmConfig(llmConfig);
78
79
  if (isMobile()) {
79
- const stripped = stripMultiGpuKeys(llmConfigStrings);
80
+ const stripped = stripMultiGpuKeys(llmConfigStrings, true);
80
81
  if (stripped.length > 0) {
81
- getEngineLogger().warn(`[${ModelType.llamacppCompletion}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
82
+ getEngineLogger().warn(`[${ModelType.llamacppCompletion}:${modelId}] Removing unsupported mobile parameters: ${stripped.join(', ')}`);
82
83
  }
83
84
  }
84
85
  // Addon overflow flag: at a single slot the addon throws when driven over
@@ -116,6 +117,7 @@ export const llmPlugin = definePlugin({
116
117
  displayName: 'LLM (llama.cpp)',
117
118
  addonPackage: ADDON_LLM,
118
119
  loadConfigSchema: llmConfigBaseSchema,
120
+ assessFit: LlmLlamacpp.assessFit,
119
121
  async resolveConfig(cfg, ctx) {
120
122
  const { projectionModelSrc, ...llmConfig } = cfg;
121
123
  if (!projectionModelSrc) {
@@ -145,7 +147,11 @@ export const llmPlugin = definePlugin({
145
147
  const dialect = request.toolDialect ?? detectToolDialect(request.modelId);
146
148
  const modelCfg = getModelConfig(request.modelId);
147
149
  const toolsActive = modelCfg.tools === true;
148
- const toolsByPosition = request.prompts.map((prompt) => prompt.tools ?? []);
150
+ // Parse against what each prompt can call: with deferred tools that is
151
+ // the rendered set plus whatever an earlier `tool_search` loaded.
152
+ const toolsByPosition = request.prompts.map((prompt) => resolveDeferredTools(prompt.tools, prompt.history)?.callableTools ??
153
+ prompt.tools?.map(toWireTool) ??
154
+ []);
149
155
  const toolsById = new Map();
150
156
  const normalizers = new Map();
151
157
  const bufferedEvents = [];