@qvac/inference 0.20.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/NOTICE +90 -24
  2. package/README.md +8 -5
  3. package/dist/api/bci-transcribe.d.ts +4 -1
  4. package/dist/api/index.d.ts +1 -0
  5. package/dist/api/index.js +1 -0
  6. package/dist/api/rpc-server.d.ts +19 -0
  7. package/dist/api/rpc-server.js +44 -0
  8. package/dist/api/text-to-speech.d.ts +9 -0
  9. package/dist/api/text-to-speech.js +9 -0
  10. package/dist/api/transcribe.d.ts +11 -7
  11. package/dist/api/transcribe.js +47 -15
  12. package/dist/api/video-request.d.ts +5 -5
  13. package/dist/dispatch.js +39 -7
  14. package/dist/errors/index.d.ts +7 -0
  15. package/dist/errors/index.js +15 -0
  16. package/dist/handlers/assess-model-fit.d.ts +8 -20
  17. package/dist/handlers/assess-model-fit.js +195 -77
  18. package/dist/handlers/completion-orchestrate.js +17 -0
  19. package/dist/handlers/get-loaded-model-info.js +4 -1
  20. package/dist/handlers/get-model-info.js +29 -1
  21. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  22. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  23. package/dist/handlers/load-model/hyperdrive.js +2 -2
  24. package/dist/handlers/load-model/registry.js +59 -0
  25. package/dist/handlers/rpc-server.d.ts +23 -0
  26. package/dist/handlers/rpc-server.js +141 -0
  27. package/dist/index.d.ts +2 -1
  28. package/dist/index.js +2 -1
  29. package/dist/models/registry/models.d.ts +7731 -5672
  30. package/dist/models/registry/models.js +4652 -2526
  31. package/dist/models/registry/resource-profiles.js +481 -94
  32. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  34. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  36. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  40. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
  41. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  42. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  43. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  44. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  45. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  46. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  47. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  48. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  49. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  50. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  51. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  52. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  53. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  54. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  55. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  56. package/dist/plugins/ops/bci-transcribe.js +18 -13
  57. package/dist/plugins/ops/load-model.js +2 -2
  58. package/dist/plugins/ops/transcribe.d.ts +4 -2
  59. package/dist/plugins/ops/transcribe.js +39 -2
  60. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  61. package/dist/plugins/ops/translate-stats.js +64 -7
  62. package/dist/plugins/ops/translate.js +20 -4
  63. package/dist/profiling/operation-metrics.js +56 -0
  64. package/dist/registry.js +4 -0
  65. package/dist/resources/model-fit/assess.d.ts +37 -2
  66. package/dist/resources/model-fit/assess.js +24 -3
  67. package/dist/resources/model-fit/estimators/llm.js +2 -2
  68. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  69. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
  70. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
  71. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  72. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  73. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  74. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  75. package/dist/resources/model-fit/floor.js +2 -2
  76. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  77. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  78. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  79. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  80. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  81. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  82. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  83. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  84. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  85. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  86. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  87. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  88. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  89. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  90. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  91. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  92. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  93. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  94. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  95. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  96. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  97. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  98. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  99. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  100. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  101. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  102. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  103. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  104. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  105. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  106. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  107. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  108. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  109. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  110. package/dist/resources/model-fit/platform.d.ts +9 -0
  111. package/dist/resources/model-fit/platform.js +28 -0
  112. package/dist/rpc/device-map.d.ts +13 -0
  113. package/dist/rpc/device-map.js +22 -0
  114. package/dist/rpc/discovery.d.ts +12 -0
  115. package/dist/rpc/discovery.js +225 -0
  116. package/dist/rpc/ggml-provider.d.ts +4 -0
  117. package/dist/rpc/ggml-provider.js +25 -0
  118. package/dist/rpc/index.d.ts +2 -0
  119. package/dist/rpc/index.js +1 -0
  120. package/dist/rpc/instance.d.ts +6 -0
  121. package/dist/rpc/instance.js +21 -0
  122. package/dist/rpc/inventory.d.ts +5 -0
  123. package/dist/rpc/inventory.js +144 -0
  124. package/dist/rpc/manager.d.ts +14 -0
  125. package/dist/rpc/manager.js +174 -0
  126. package/dist/rpc/network.d.ts +11 -0
  127. package/dist/rpc/network.js +85 -0
  128. package/dist/rpc/provider.d.ts +6 -0
  129. package/dist/rpc/provider.js +24 -0
  130. package/dist/runtime/lifecycle.js +31 -13
  131. package/dist/runtime/request-context.d.ts +1 -1
  132. package/dist/runtime/state.js +5 -0
  133. package/dist/schemas/assess-model-fit.d.ts +269 -85
  134. package/dist/schemas/assess-model-fit.js +78 -54
  135. package/dist/schemas/audio-gen.d.ts +6 -6
  136. package/dist/schemas/audio-gen.js +2 -2
  137. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  138. package/dist/schemas/batch-completion-stream.js +5 -3
  139. package/dist/schemas/bci-config.js +1 -1
  140. package/dist/schemas/bci.d.ts +86 -1
  141. package/dist/schemas/bci.js +21 -2
  142. package/dist/schemas/cancel.d.ts +10 -0
  143. package/dist/schemas/cancel.js +2 -0
  144. package/dist/schemas/common.d.ts +708 -66
  145. package/dist/schemas/common.js +7 -0
  146. package/dist/schemas/completion-event.d.ts +6 -6
  147. package/dist/schemas/completion-stream.d.ts +27 -8
  148. package/dist/schemas/completion-stream.js +41 -4
  149. package/dist/schemas/config.d.ts +104 -36
  150. package/dist/schemas/config.js +10 -0
  151. package/dist/schemas/embed.d.ts +2 -2
  152. package/dist/schemas/error.js +13 -0
  153. package/dist/schemas/errors.d.ts +1 -0
  154. package/dist/schemas/errors.js +5 -0
  155. package/dist/schemas/index.d.ts +1 -0
  156. package/dist/schemas/index.js +1 -0
  157. package/dist/schemas/llamacpp-config.d.ts +25 -18
  158. package/dist/schemas/llamacpp-config.js +26 -16
  159. package/dist/schemas/load-model.d.ts +2608 -469
  160. package/dist/schemas/load-model.js +2 -2
  161. package/dist/schemas/mcp-adapter.d.ts +4 -0
  162. package/dist/schemas/model-config-schemas.d.ts +183 -6
  163. package/dist/schemas/model-info.d.ts +56 -0
  164. package/dist/schemas/model-info.js +5 -1
  165. package/dist/schemas/plugin.d.ts +8 -2
  166. package/dist/schemas/profiling.d.ts +4 -4
  167. package/dist/schemas/reload-config.d.ts +30 -0
  168. package/dist/schemas/rpc-server.d.ts +101 -0
  169. package/dist/schemas/rpc-server.js +87 -0
  170. package/dist/schemas/sdcpp-config.d.ts +7 -3
  171. package/dist/schemas/sdcpp-config.js +37 -2
  172. package/dist/schemas/system-resources.d.ts +4 -4
  173. package/dist/schemas/text-to-speech.d.ts +529 -1
  174. package/dist/schemas/text-to-speech.js +122 -12
  175. package/dist/schemas/tools.d.ts +2 -0
  176. package/dist/schemas/tools.js +8 -0
  177. package/dist/schemas/transcription-config.d.ts +5 -0
  178. package/dist/schemas/transcription-config.js +20 -2
  179. package/dist/schemas/transcription.d.ts +164 -0
  180. package/dist/schemas/transcription.js +71 -4
  181. package/dist/schemas/vla.d.ts +1 -1
  182. package/dist/surface.d.ts +4 -1
  183. package/dist/surface.js +3 -1
  184. package/dist/utils/addon-responses.d.ts +5 -1
  185. package/dist/utils/archive.js +5 -1
  186. package/dist/utils/asr-diagnostics.d.ts +12 -0
  187. package/dist/utils/asr-diagnostics.js +42 -0
  188. package/dist/utils/asr-events.d.ts +1 -0
  189. package/dist/utils/asr-events.js +3 -2
  190. package/dist/utils/cache/paths.js +3 -2
  191. package/dist/utils/mcp-adapter.js +7 -1
  192. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  193. package/dist/utils/multi-gpu-mobile.js +8 -6
  194. package/dist/utils/path-security.d.ts +5 -0
  195. package/dist/utils/path-security.js +14 -0
  196. package/dist/utils/tool-helpers.d.ts +4 -0
  197. package/dist/utils/tool-helpers.js +2 -0
  198. package/dist/utils/tools/defer.d.ts +93 -0
  199. package/dist/utils/tools/defer.js +290 -0
  200. package/dist/utils/tools/index.d.ts +1 -0
  201. package/dist/utils/tools/index.js +1 -0
  202. package/dist/utils/transcribe-metadata.d.ts +21 -7
  203. package/dist/utils/transcribe-metadata.js +24 -8
  204. package/dist/utils/tts-stats.d.ts +6 -0
  205. package/dist/utils/tts-stats.js +14 -1
  206. package/package.json +43 -29
  207. package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
  208. package/dist/resources/model-fit/fit-stub/index.js +0 -2
  209. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
  210. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
  211. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
@@ -34,6 +34,17 @@ function hasTypedFields(error) {
34
34
  function isQvacError(error) {
35
35
  return error instanceof QvacErrorBase;
36
36
  }
37
+ function serializeCause(cause, depth = 0) {
38
+ if (!(cause instanceof Error))
39
+ return typeof cause === 'string' ? cause : String(cause);
40
+ return {
41
+ name: cause.name,
42
+ message: cause.message,
43
+ stack: cause.stack,
44
+ ...('code' in cause && typeof cause.code === 'number' && { code: cause.code }),
45
+ ...(cause.cause !== undefined && depth < 5 && { cause: serializeCause(cause.cause, depth + 1) })
46
+ };
47
+ }
37
48
  export function createErrorResponse(error) {
38
49
  if (isQvacError(error)) {
39
50
  const qvacData = error.toJSON();
@@ -45,6 +56,8 @@ export function createErrorResponse(error) {
45
56
  stack: qvacData.stack,
46
57
  timestamp: new Date().toISOString()
47
58
  };
59
+ if (qvacData.cause !== undefined)
60
+ response.cause = serializeCause(qvacData.cause);
48
61
  if (hasTypedFields(error)) {
49
62
  response.typedFields = error.toErrorResponseFields();
50
63
  }
@@ -28,6 +28,7 @@ export declare const ERROR_CODES: {
28
28
  readonly TTS_REFERENCE_AUDIO_REQUIRED: 52209;
29
29
  readonly LEGACY_PARAKEET_MODEL_DEPRECATED: 52210;
30
30
  readonly LEGACY_TTS_MODEL_DEPRECATED: 52211;
31
+ readonly RPC_SERVER_OPERATION_FAILED: 52423;
31
32
  readonly MODEL_UNLOAD_FAILED: 52400;
32
33
  readonly EMBED_FAILED: 52401;
33
34
  readonly EMBED_NO_EMBEDDINGS: 52402;
@@ -37,6 +37,7 @@ export const ERROR_CODES = {
37
37
  LEGACY_PARAKEET_MODEL_DEPRECATED: 52210,
38
38
  LEGACY_TTS_MODEL_DEPRECATED: 52211,
39
39
  // Model operations (52,400-52,799)
40
+ RPC_SERVER_OPERATION_FAILED: 52423,
40
41
  MODEL_UNLOAD_FAILED: 52400,
41
42
  EMBED_FAILED: 52401,
42
43
  EMBED_NO_EMBEDDINGS: 52402,
@@ -209,6 +210,10 @@ const errorDefinitions = {
209
210
  name: 'UNKNOWN_MODEL_TYPE',
210
211
  message: (modelType) => `Unknown model type: ${modelType}. Register the plugin for "${modelType}" in code with \`registerPlugin\` / \`plugins([...])\` before this call.`
211
212
  },
213
+ [ERROR_CODES.RPC_SERVER_OPERATION_FAILED]: {
214
+ name: 'RPC_SERVER_OPERATION_FAILED',
215
+ message: (operation, details) => `${operation}: ${details}`
216
+ },
212
217
  // Model loading
213
218
  [ERROR_CODES.MODEL_LOAD_FAILED]: {
214
219
  name: 'MODEL_LOAD_FAILED',
@@ -46,3 +46,4 @@ export * from './plugin.js';
46
46
  export * from './registry.js';
47
47
  export * from './model-resource-profile.js';
48
48
  export * from './assess-model-fit.js';
49
+ export * from './rpc-server.js';
@@ -46,3 +46,4 @@ export * from './plugin.js';
46
46
  export * from './registry.js';
47
47
  export * from './model-resource-profile.js';
48
48
  export * from './assess-model-fit.js';
49
+ export * from './rpc-server.js';
@@ -47,15 +47,17 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
47
47
  integrated: "integrated";
48
48
  dedicated: "dedicated";
49
49
  }>]>>;
50
+ 'rpc-servers': z.ZodOptional<z.ZodString>;
51
+ devices: z.ZodOptional<z.ZodString>;
50
52
  'split-mode': z.ZodOptional<z.ZodEnum<{
51
53
  none: "none";
52
54
  layer: "layer";
53
55
  tensor: "tensor";
54
56
  }>>;
55
57
  'flash-attn': z.ZodOptional<z.ZodEnum<{
58
+ auto: "auto";
56
59
  on: "on";
57
60
  off: "off";
58
- auto: "auto";
59
61
  }>>;
60
62
  'tensor-split': z.ZodOptional<z.ZodString>;
61
63
  'batch-size': z.ZodOptional<z.ZodNumber>;
@@ -74,9 +76,9 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
74
76
  'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
75
77
  'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
76
78
  'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
79
+ auto: "auto";
77
80
  on: "on";
78
81
  off: "off";
79
- auto: "auto";
80
82
  }>>;
81
83
  fit: z.ZodOptional<z.ZodBoolean>;
82
84
  'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
@@ -136,7 +138,6 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
136
138
  export type LlmConfigInput = z.infer<typeof llmConfigBaseSchema>;
137
139
  export declare const LLM_CONFIG_DEFAULTS: {
138
140
  readonly ctx_size: 1024;
139
- readonly gpu_layers: 99;
140
141
  readonly device: "gpu";
141
142
  readonly system_prompt: "You are a helpful assistant.";
142
143
  readonly image_tile_mode: "sequential";
@@ -177,15 +178,17 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
177
178
  integrated: "integrated";
178
179
  dedicated: "dedicated";
179
180
  }>]>>;
181
+ 'rpc-servers': z.ZodOptional<z.ZodString>;
182
+ devices: z.ZodOptional<z.ZodString>;
180
183
  'split-mode': z.ZodOptional<z.ZodEnum<{
181
184
  none: "none";
182
185
  layer: "layer";
183
186
  tensor: "tensor";
184
187
  }>>;
185
188
  'flash-attn': z.ZodOptional<z.ZodEnum<{
189
+ auto: "auto";
186
190
  on: "on";
187
191
  off: "off";
188
- auto: "auto";
189
192
  }>>;
190
193
  'tensor-split': z.ZodOptional<z.ZodString>;
191
194
  'batch-size': z.ZodOptional<z.ZodNumber>;
@@ -204,9 +207,9 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
204
207
  'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
205
208
  'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
206
209
  'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
210
+ auto: "auto";
207
211
  on: "on";
208
212
  off: "off";
209
- auto: "auto";
210
213
  }>>;
211
214
  fit: z.ZodOptional<z.ZodBoolean>;
212
215
  'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
@@ -268,7 +271,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
268
271
  top_p?: number | undefined;
269
272
  top_k?: number | undefined;
270
273
  seed?: number | undefined;
271
- gpu_layers: number | undefined;
274
+ gpu_layers?: number | undefined;
272
275
  lora?: string | undefined;
273
276
  device: string | undefined;
274
277
  predict?: number | undefined;
@@ -284,8 +287,10 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
284
287
  'cache-type-k'?: string | undefined;
285
288
  'cache-type-v'?: string | undefined;
286
289
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
290
+ 'rpc-servers'?: string | undefined;
291
+ devices?: string | undefined;
287
292
  'split-mode'?: "none" | "layer" | "tensor" | undefined;
288
- 'flash-attn'?: "on" | "off" | "auto" | undefined;
293
+ 'flash-attn'?: "auto" | "on" | "off" | undefined;
289
294
  'tensor-split'?: string | undefined;
290
295
  'batch-size'?: number | undefined;
291
296
  'ubatch-size'?: number | undefined;
@@ -302,7 +307,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
302
307
  'n-cpu-ffn'?: number | undefined;
303
308
  'moe-cache-mib'?: number | "auto" | undefined;
304
309
  'prefetch-weights'?: boolean | "auto" | undefined;
305
- 'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
310
+ 'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
306
311
  fit?: boolean | undefined;
307
312
  'fit-target'?: string | number | undefined;
308
313
  'fit-ctx'?: number | undefined;
@@ -346,8 +351,10 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
346
351
  'cache-type-k'?: string | undefined;
347
352
  'cache-type-v'?: string | undefined;
348
353
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
354
+ 'rpc-servers'?: string | undefined;
355
+ devices?: string | undefined;
349
356
  'split-mode'?: "none" | "layer" | "tensor" | undefined;
350
- 'flash-attn'?: "on" | "off" | "auto" | undefined;
357
+ 'flash-attn'?: "auto" | "on" | "off" | undefined;
351
358
  'tensor-split'?: string | undefined;
352
359
  'batch-size'?: number | undefined;
353
360
  'ubatch-size'?: number | undefined;
@@ -364,7 +371,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
364
371
  'n-cpu-ffn'?: number | undefined;
365
372
  'moe-cache-mib'?: number | "auto" | undefined;
366
373
  'prefetch-weights'?: boolean | "auto" | undefined;
367
- 'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
374
+ 'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
368
375
  fit?: boolean | undefined;
369
376
  'fit-target'?: string | number | undefined;
370
377
  'fit-ctx'?: number | undefined;
@@ -391,8 +398,8 @@ export type LlmConfig = z.infer<typeof llmConfigSchema>;
391
398
  export declare const embedConfigBaseSchema: z.ZodObject<{
392
399
  gpuLayers: z.ZodOptional<z.ZodNumber>;
393
400
  device: z.ZodOptional<z.ZodEnum<{
394
- gpu: "gpu";
395
401
  cpu: "cpu";
402
+ gpu: "gpu";
396
403
  }>>;
397
404
  batchSize: z.ZodOptional<z.ZodNumber>;
398
405
  pooling: z.ZodOptional<z.ZodEnum<{
@@ -408,9 +415,9 @@ export declare const embedConfigBaseSchema: z.ZodObject<{
408
415
  }>>;
409
416
  embdNormalize: z.ZodOptional<z.ZodNumber>;
410
417
  flashAttention: z.ZodOptional<z.ZodEnum<{
418
+ auto: "auto";
411
419
  on: "on";
412
420
  off: "off";
413
- auto: "auto";
414
421
  }>>;
415
422
  mainGpu: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
416
423
  integrated: "integrated";
@@ -438,8 +445,8 @@ export declare const EMBED_CONFIG_DEFAULTS: {
438
445
  export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
439
446
  gpuLayers: z.ZodOptional<z.ZodNumber>;
440
447
  device: z.ZodOptional<z.ZodEnum<{
441
- gpu: "gpu";
442
448
  cpu: "cpu";
449
+ gpu: "gpu";
443
450
  }>>;
444
451
  batchSize: z.ZodOptional<z.ZodNumber>;
445
452
  pooling: z.ZodOptional<z.ZodEnum<{
@@ -455,9 +462,9 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
455
462
  }>>;
456
463
  embdNormalize: z.ZodOptional<z.ZodNumber>;
457
464
  flashAttention: z.ZodOptional<z.ZodEnum<{
465
+ auto: "auto";
458
466
  on: "on";
459
467
  off: "off";
460
- auto: "auto";
461
468
  }>>;
462
469
  mainGpu: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
463
470
  integrated: "integrated";
@@ -477,12 +484,12 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
477
484
  openclCacheDir: z.ZodOptional<z.ZodString>;
478
485
  }, z.core.$strip>, z.ZodTransform<{
479
486
  gpuLayers: number | undefined;
480
- device: "gpu" | "cpu" | undefined;
487
+ device: "cpu" | "gpu" | undefined;
481
488
  batchSize: number | undefined;
482
489
  pooling?: "none" | "mean" | "cls" | "last" | "rank" | undefined;
483
490
  attention?: "causal" | "non-causal" | undefined;
484
491
  embdNormalize?: number | undefined;
485
- flashAttention?: "on" | "off" | "auto" | undefined;
492
+ flashAttention?: "auto" | "on" | "off" | undefined;
486
493
  mainGpu?: number | "integrated" | "dedicated" | undefined;
487
494
  splitMode?: "none" | "layer" | undefined;
488
495
  tensorSplit?: string | undefined;
@@ -490,12 +497,12 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
490
497
  openclCacheDir?: string | undefined;
491
498
  }, {
492
499
  gpuLayers?: number | undefined;
493
- device?: "gpu" | "cpu" | undefined;
500
+ device?: "cpu" | "gpu" | undefined;
494
501
  batchSize?: number | undefined;
495
502
  pooling?: "none" | "mean" | "cls" | "last" | "rank" | undefined;
496
503
  attention?: "causal" | "non-causal" | undefined;
497
504
  embdNormalize?: number | undefined;
498
- flashAttention?: "on" | "off" | "auto" | undefined;
505
+ flashAttention?: "auto" | "on" | "off" | undefined;
499
506
  mainGpu?: number | "integrated" | "dedicated" | undefined;
500
507
  splitMode?: "none" | "layer" | undefined;
501
508
  tensorSplit?: string | undefined;
@@ -37,7 +37,7 @@ export const llmConfigBaseSchema = z.object({
37
37
  gpu_layers: z
38
38
  .number()
39
39
  .optional()
40
- .describe('Number of model layers to offload to the GPU. Default 99 (offload all).'),
40
+ .describe('Number of model layers to offload to the GPU. Unset by default, which lets the runtime fit the placement to free device memory (offloading every layer when it fits). Setting it pins the layer count and disables that fit.'),
41
41
  lora: z
42
42
  .string()
43
43
  .optional()
@@ -103,10 +103,24 @@ export const llmConfigBaseSchema = z.object({
103
103
  .union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])])
104
104
  .optional()
105
105
  .describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
106
+ 'rpc-servers': z
107
+ .string()
108
+ .min(1)
109
+ .refine((value) => {
110
+ const endpoints = value.split(',').map((endpoint) => endpoint.trim());
111
+ return new Set(endpoints).size === endpoints.length;
112
+ }, 'RPC server endpoints must be unique')
113
+ .optional()
114
+ .describe('Comma-separated unique native RPC host:port endpoints in device registration order for this load. Requires compatible RPC addon builds and a trusted private network; native traffic is unencrypted and unauthenticated.'),
115
+ devices: z
116
+ .string()
117
+ .min(1)
118
+ .optional()
119
+ .describe("Explicit ordered native device names, for example RPC0,RPC1. RPC indices enumerate every device in this load's rpc-servers endpoint order, starting at RPC0 for each load. Pass exactly those endpoints in that order to getRpcDeviceMap, including on reused workers. tensor-split weights follow the selected devices order."),
106
120
  'split-mode': z
107
121
  .enum(['none', 'layer', 'tensor'])
108
122
  .optional()
109
- .describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across all visible GPUs; desktop-only, requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
123
+ .describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across selected local or RPC GPUs; requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
110
124
  'flash-attn': z
111
125
  .enum(['on', 'off', 'auto'])
112
126
  .optional()
@@ -196,7 +210,7 @@ export const llmConfigBaseSchema = z.object({
196
210
  fit: z
197
211
  .boolean()
198
212
  .optional()
199
- .describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting this field releases the `gpu_layers` default so fit can choose it, which offloads every layer rather than 99 if fit then gives up; setting `gpu_layers` alongside it pins the value and makes fit abort, leaving every argument unchanged. Incompatible with `split-mode: tensor`, which disables it.'),
213
+ .describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting `gpu_layers` pins the layer count and makes fit abort, leaving every argument unchanged. Incompatible with `split-mode: tensor`, which disables it.'),
200
214
  'fit-target': z
201
215
  .union([z.number().int().min(0), z.string().regex(/^\d+(,\d+)*$/)])
202
216
  .optional()
@@ -235,26 +249,22 @@ export const llmConfigBaseSchema = z.object({
235
249
  .describe('Run the multimodal projector (mmproj / vision encoder) on the GPU (multimodal models only). `true` forces GPU, `false` forces CPU. When unset, the backend is auto-selected per device class: GPU on desktop/iOS and Android Adreno 800+; CPU on every other Android GPU (Arm Mali, Adreno <800, and undetectable Adreno tiers), with the LLM layers still on the GPU. Only honoured when the model itself runs on a GPU backend — ignored with a warning on CPU.')
236
250
  });
237
251
  // Default values - typed as partial of the config
252
+ //
253
+ // gpu_layers carries no default: any value reads as user intent to qvac-fabric,
254
+ // which then aborts its fit instead of placing layers to free device memory.
255
+ // Absent emits no `--gpu-layers`, so the addon keeps fabric's own -1 — already
256
+ // "every layer" whenever the fit does not run.
238
257
  export const LLM_CONFIG_DEFAULTS = {
239
258
  ctx_size: 1024,
240
- gpu_layers: 99,
241
259
  device: 'gpu',
242
260
  system_prompt: 'You are a helpful assistant.',
243
261
  image_tile_mode: 'sequential'
244
262
  };
245
- /**
246
- * `llama_model_default_params().n_gpu_layers`. Fabric's fit aborts and rolls
247
- * back every argument it had computed when `n_gpu_layers` holds any other
248
- * value, so a caller opting into `fit` gets this in place of the default above.
249
- */
250
- const LLAMA_GPU_LAYERS_DEFAULT = -1;
251
263
  // Full schema - applies defaults via transform (no duplication)
252
- export const llmConfigSchema = llmConfigBaseSchema.transform((data) => {
253
- const merged = { ...LLM_CONFIG_DEFAULTS, ...data };
254
- return data.fit !== undefined && data.gpu_layers === undefined
255
- ? { ...merged, gpu_layers: LLAMA_GPU_LAYERS_DEFAULT }
256
- : merged;
257
- });
264
+ export const llmConfigSchema = llmConfigBaseSchema.transform((data) => ({
265
+ ...LLM_CONFIG_DEFAULTS,
266
+ ...data
267
+ }));
258
268
  // Base schema - validates types, all fields optional (for input validation)
259
269
  export const embedConfigBaseSchema = z.object({
260
270
  gpuLayers: z