@qvac/inference 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/NOTICE +107 -33
  2. package/README.md +8 -5
  3. package/dist/api/assess-model-fit.d.ts +9 -4
  4. package/dist/api/assess-model-fit.js +9 -4
  5. package/dist/api/bci-transcribe.d.ts +4 -1
  6. package/dist/api/index.d.ts +1 -0
  7. package/dist/api/index.js +1 -0
  8. package/dist/api/rpc-server.d.ts +19 -0
  9. package/dist/api/rpc-server.js +44 -0
  10. package/dist/api/text-to-speech.d.ts +9 -0
  11. package/dist/api/text-to-speech.js +9 -0
  12. package/dist/api/transcribe.d.ts +11 -7
  13. package/dist/api/transcribe.js +47 -15
  14. package/dist/api/video-request.d.ts +5 -5
  15. package/dist/dispatch.js +39 -7
  16. package/dist/errors/index.d.ts +7 -0
  17. package/dist/errors/index.js +15 -0
  18. package/dist/handlers/assess-model-fit.d.ts +18 -6
  19. package/dist/handlers/assess-model-fit.js +215 -35
  20. package/dist/handlers/completion-orchestrate.js +17 -0
  21. package/dist/handlers/get-loaded-model-info.js +4 -1
  22. package/dist/handlers/get-model-info.js +29 -1
  23. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  24. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  25. package/dist/handlers/load-model/hyperdrive.js +2 -2
  26. package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
  27. package/dist/handlers/load-model/registry-download-utils.js +2 -1
  28. package/dist/handlers/load-model/registry.js +59 -0
  29. package/dist/handlers/rpc-server.d.ts +23 -0
  30. package/dist/handlers/rpc-server.js +141 -0
  31. package/dist/index.d.ts +2 -1
  32. package/dist/index.js +2 -1
  33. package/dist/models/registry/models.d.ts +7731 -5672
  34. package/dist/models/registry/models.js +4652 -2526
  35. package/dist/models/registry/resource-profiles.js +481 -94
  36. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  37. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  38. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  39. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  40. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  41. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  44. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
  45. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  46. package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
  47. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  48. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  49. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  50. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  51. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  52. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  54. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  55. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  56. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  57. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  58. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  59. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  60. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  61. package/dist/plugins/ops/bci-transcribe.js +18 -13
  62. package/dist/plugins/ops/load-model.js +2 -2
  63. package/dist/plugins/ops/transcribe.d.ts +4 -2
  64. package/dist/plugins/ops/transcribe.js +39 -2
  65. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  66. package/dist/plugins/ops/translate-stats.js +64 -7
  67. package/dist/plugins/ops/translate.js +20 -4
  68. package/dist/profiling/operation-metrics.js +56 -0
  69. package/dist/registry.js +4 -0
  70. package/dist/resources/model-fit/assess.d.ts +46 -2
  71. package/dist/resources/model-fit/assess.js +67 -3
  72. package/dist/resources/model-fit/estimators/llm.js +2 -2
  73. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  74. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
  75. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
  76. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  77. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  78. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  79. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  80. package/dist/resources/model-fit/floor.js +2 -2
  81. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  82. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  83. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  84. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  85. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  86. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  87. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  88. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  89. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  90. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  91. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  92. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  93. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  94. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  95. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  96. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  97. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  98. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  99. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  100. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  101. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  102. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  103. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  104. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  105. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  106. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  107. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  108. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  109. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  110. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  111. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  112. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  113. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  114. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  115. package/dist/resources/model-fit/platform.d.ts +9 -0
  116. package/dist/resources/model-fit/platform.js +28 -0
  117. package/dist/rpc/device-map.d.ts +13 -0
  118. package/dist/rpc/device-map.js +22 -0
  119. package/dist/rpc/discovery.d.ts +12 -0
  120. package/dist/rpc/discovery.js +225 -0
  121. package/dist/rpc/ggml-provider.d.ts +4 -0
  122. package/dist/rpc/ggml-provider.js +25 -0
  123. package/dist/rpc/index.d.ts +2 -0
  124. package/dist/rpc/index.js +1 -0
  125. package/dist/rpc/instance.d.ts +6 -0
  126. package/dist/rpc/instance.js +21 -0
  127. package/dist/rpc/inventory.d.ts +5 -0
  128. package/dist/rpc/inventory.js +144 -0
  129. package/dist/rpc/manager.d.ts +14 -0
  130. package/dist/rpc/manager.js +174 -0
  131. package/dist/rpc/network.d.ts +11 -0
  132. package/dist/rpc/network.js +85 -0
  133. package/dist/rpc/provider.d.ts +6 -0
  134. package/dist/rpc/provider.js +24 -0
  135. package/dist/runtime/lifecycle.js +31 -13
  136. package/dist/runtime/request-context.d.ts +1 -1
  137. package/dist/runtime/state.js +5 -0
  138. package/dist/schemas/assess-model-fit.d.ts +284 -77
  139. package/dist/schemas/assess-model-fit.js +93 -55
  140. package/dist/schemas/audio-gen.d.ts +6 -6
  141. package/dist/schemas/audio-gen.js +2 -2
  142. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  143. package/dist/schemas/batch-completion-stream.js +5 -3
  144. package/dist/schemas/bci-config.js +1 -1
  145. package/dist/schemas/bci.d.ts +86 -1
  146. package/dist/schemas/bci.js +21 -2
  147. package/dist/schemas/cancel.d.ts +10 -0
  148. package/dist/schemas/cancel.js +2 -0
  149. package/dist/schemas/common.d.ts +765 -56
  150. package/dist/schemas/common.js +7 -0
  151. package/dist/schemas/completion-event.d.ts +6 -6
  152. package/dist/schemas/completion-stream.d.ts +27 -8
  153. package/dist/schemas/completion-stream.js +41 -4
  154. package/dist/schemas/config.d.ts +236 -30
  155. package/dist/schemas/config.js +10 -0
  156. package/dist/schemas/embed.d.ts +2 -2
  157. package/dist/schemas/error.js +13 -0
  158. package/dist/schemas/errors.d.ts +1 -0
  159. package/dist/schemas/errors.js +5 -0
  160. package/dist/schemas/index.d.ts +1 -0
  161. package/dist/schemas/index.js +1 -0
  162. package/dist/schemas/llamacpp-config.d.ts +105 -14
  163. package/dist/schemas/llamacpp-config.js +113 -3
  164. package/dist/schemas/load-model.d.ts +3102 -627
  165. package/dist/schemas/load-model.js +2 -2
  166. package/dist/schemas/mcp-adapter.d.ts +4 -0
  167. package/dist/schemas/model-config-schemas.d.ts +205 -5
  168. package/dist/schemas/model-info.d.ts +56 -0
  169. package/dist/schemas/model-info.js +5 -1
  170. package/dist/schemas/plugin.d.ts +8 -2
  171. package/dist/schemas/profiling.d.ts +4 -4
  172. package/dist/schemas/reload-config.d.ts +30 -0
  173. package/dist/schemas/rpc-server.d.ts +101 -0
  174. package/dist/schemas/rpc-server.js +87 -0
  175. package/dist/schemas/sdcpp-config.d.ts +7 -3
  176. package/dist/schemas/sdcpp-config.js +37 -2
  177. package/dist/schemas/system-resources.d.ts +4 -4
  178. package/dist/schemas/text-to-speech.d.ts +529 -1
  179. package/dist/schemas/text-to-speech.js +122 -12
  180. package/dist/schemas/tools.d.ts +2 -0
  181. package/dist/schemas/tools.js +8 -0
  182. package/dist/schemas/transcription-config.d.ts +5 -0
  183. package/dist/schemas/transcription-config.js +20 -2
  184. package/dist/schemas/transcription.d.ts +164 -0
  185. package/dist/schemas/transcription.js +71 -4
  186. package/dist/schemas/vla.d.ts +1 -1
  187. package/dist/surface.d.ts +4 -1
  188. package/dist/surface.js +3 -1
  189. package/dist/utils/addon-responses.d.ts +5 -1
  190. package/dist/utils/archive.js +5 -1
  191. package/dist/utils/asr-diagnostics.d.ts +12 -0
  192. package/dist/utils/asr-diagnostics.js +42 -0
  193. package/dist/utils/asr-events.d.ts +1 -0
  194. package/dist/utils/asr-events.js +3 -2
  195. package/dist/utils/cache/paths.js +3 -2
  196. package/dist/utils/mcp-adapter.js +7 -1
  197. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  198. package/dist/utils/multi-gpu-mobile.js +8 -6
  199. package/dist/utils/path-security.d.ts +5 -0
  200. package/dist/utils/path-security.js +14 -0
  201. package/dist/utils/tool-helpers.d.ts +4 -0
  202. package/dist/utils/tool-helpers.js +2 -0
  203. package/dist/utils/tools/defer.d.ts +93 -0
  204. package/dist/utils/tools/defer.js +290 -0
  205. package/dist/utils/tools/index.d.ts +1 -0
  206. package/dist/utils/tools/index.js +1 -0
  207. package/dist/utils/transcribe-metadata.d.ts +21 -7
  208. package/dist/utils/transcribe-metadata.js +24 -8
  209. package/dist/utils/tts-stats.d.ts +6 -0
  210. package/dist/utils/tts-stats.js +14 -1
  211. package/package.json +44 -30
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  213. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
@@ -166,6 +166,14 @@ export const qvacConfigSchema = z.object({
166
166
  * Defaults to 60000 (60 seconds).
167
167
  */
168
168
  registryStreamTimeoutMs: z.number().int().positive().optional(),
169
+ /**
170
+ * Budget in milliseconds for the registry fetch of a model's weightless
171
+ * description during `assessModelFit`. One budget covers the lookup and the
172
+ * whole companion set; past it the assessment falls back to the computed
173
+ * floor. Raise this on slow or high-latency connections.
174
+ * Defaults to 40000 (40 seconds).
175
+ */
176
+ fitStubBudgetMs: z.number().int().positive().optional(),
169
177
  /**
170
178
  * Device-specific config defaults.
171
179
  * Use this to override model config defaults for specific devices.
@@ -203,6 +211,8 @@ export const qvacConfigSchema = z.object({
203
211
  * ending in `/plugin`.
204
212
  */
205
213
  plugins: z.array(z.string()).optional(),
214
+ /** Optional provider module selected at SDK bundle time. Direct Bare callers register in code. */
215
+ rpcServerProvider: z.string().min(1).optional(),
206
216
  /**
207
217
  * Inert: we have no bundling or ABI-verification step, so this field is
208
218
  * unused. Carried over from the bundle-time config schema.
@@ -14,8 +14,8 @@ export declare const embedStatsSchema: z.ZodObject<{
14
14
  tokensPerSecond: z.ZodOptional<z.ZodNumber>;
15
15
  totalTokens: z.ZodOptional<z.ZodNumber>;
16
16
  backendDevice: z.ZodOptional<z.ZodEnum<{
17
- gpu: "gpu";
18
17
  cpu: "cpu";
18
+ gpu: "gpu";
19
19
  }>>;
20
20
  contextSize: z.ZodOptional<z.ZodNumber>;
21
21
  }, z.core.$strip>;
@@ -28,8 +28,8 @@ export declare const embedResponseSchema: z.ZodObject<{
28
28
  tokensPerSecond: z.ZodOptional<z.ZodNumber>;
29
29
  totalTokens: z.ZodOptional<z.ZodNumber>;
30
30
  backendDevice: z.ZodOptional<z.ZodEnum<{
31
- gpu: "gpu";
32
31
  cpu: "cpu";
32
+ gpu: "gpu";
33
33
  }>>;
34
34
  contextSize: z.ZodOptional<z.ZodNumber>;
35
35
  }, z.core.$strip>>;
@@ -34,6 +34,17 @@ function hasTypedFields(error) {
34
34
  function isQvacError(error) {
35
35
  return error instanceof QvacErrorBase;
36
36
  }
37
+ function serializeCause(cause, depth = 0) {
38
+ if (!(cause instanceof Error))
39
+ return typeof cause === 'string' ? cause : String(cause);
40
+ return {
41
+ name: cause.name,
42
+ message: cause.message,
43
+ stack: cause.stack,
44
+ ...('code' in cause && typeof cause.code === 'number' && { code: cause.code }),
45
+ ...(cause.cause !== undefined && depth < 5 && { cause: serializeCause(cause.cause, depth + 1) })
46
+ };
47
+ }
37
48
  export function createErrorResponse(error) {
38
49
  if (isQvacError(error)) {
39
50
  const qvacData = error.toJSON();
@@ -45,6 +56,8 @@ export function createErrorResponse(error) {
45
56
  stack: qvacData.stack,
46
57
  timestamp: new Date().toISOString()
47
58
  };
59
+ if (qvacData.cause !== undefined)
60
+ response.cause = serializeCause(qvacData.cause);
48
61
  if (hasTypedFields(error)) {
49
62
  response.typedFields = error.toErrorResponseFields();
50
63
  }
@@ -28,6 +28,7 @@ export declare const ERROR_CODES: {
28
28
  readonly TTS_REFERENCE_AUDIO_REQUIRED: 52209;
29
29
  readonly LEGACY_PARAKEET_MODEL_DEPRECATED: 52210;
30
30
  readonly LEGACY_TTS_MODEL_DEPRECATED: 52211;
31
+ readonly RPC_SERVER_OPERATION_FAILED: 52423;
31
32
  readonly MODEL_UNLOAD_FAILED: 52400;
32
33
  readonly EMBED_FAILED: 52401;
33
34
  readonly EMBED_NO_EMBEDDINGS: 52402;
@@ -37,6 +37,7 @@ export const ERROR_CODES = {
37
37
  LEGACY_PARAKEET_MODEL_DEPRECATED: 52210,
38
38
  LEGACY_TTS_MODEL_DEPRECATED: 52211,
39
39
  // Model operations (52,400-52,799)
40
+ RPC_SERVER_OPERATION_FAILED: 52423,
40
41
  MODEL_UNLOAD_FAILED: 52400,
41
42
  EMBED_FAILED: 52401,
42
43
  EMBED_NO_EMBEDDINGS: 52402,
@@ -209,6 +210,10 @@ const errorDefinitions = {
209
210
  name: 'UNKNOWN_MODEL_TYPE',
210
211
  message: (modelType) => `Unknown model type: ${modelType}. Register the plugin for "${modelType}" in code with \`registerPlugin\` / \`plugins([...])\` before this call.`
211
212
  },
213
+ [ERROR_CODES.RPC_SERVER_OPERATION_FAILED]: {
214
+ name: 'RPC_SERVER_OPERATION_FAILED',
215
+ message: (operation, details) => `${operation}: ${details}`
216
+ },
212
217
  // Model loading
213
218
  [ERROR_CODES.MODEL_LOAD_FAILED]: {
214
219
  name: 'MODEL_LOAD_FAILED',
@@ -46,3 +46,4 @@ export * from './plugin.js';
46
46
  export * from './registry.js';
47
47
  export * from './model-resource-profile.js';
48
48
  export * from './assess-model-fit.js';
49
+ export * from './rpc-server.js';
@@ -46,3 +46,4 @@ export * from './plugin.js';
46
46
  export * from './registry.js';
47
47
  export * from './model-resource-profile.js';
48
48
  export * from './assess-model-fit.js';
49
+ export * from './rpc-server.js';
@@ -47,17 +47,42 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
47
47
  integrated: "integrated";
48
48
  dedicated: "dedicated";
49
49
  }>]>>;
50
+ 'rpc-servers': z.ZodOptional<z.ZodString>;
51
+ devices: z.ZodOptional<z.ZodString>;
50
52
  'split-mode': z.ZodOptional<z.ZodEnum<{
51
53
  none: "none";
52
54
  layer: "layer";
53
55
  tensor: "tensor";
54
56
  }>>;
55
57
  'flash-attn': z.ZodOptional<z.ZodEnum<{
58
+ auto: "auto";
56
59
  on: "on";
57
60
  off: "off";
58
- auto: "auto";
59
61
  }>>;
60
62
  'tensor-split': z.ZodOptional<z.ZodString>;
63
+ 'batch-size': z.ZodOptional<z.ZodNumber>;
64
+ 'ubatch-size': z.ZodOptional<z.ZodNumber>;
65
+ 'cpu-moe': z.ZodOptional<z.ZodBoolean>;
66
+ 'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
67
+ 'kv-offload': z.ZodOptional<z.ZodBoolean>;
68
+ 'image-max-tokens': z.ZodOptional<z.ZodNumber>;
69
+ 'image-min-tokens': z.ZodOptional<z.ZodNumber>;
70
+ threads: z.ZodOptional<z.ZodNumber>;
71
+ 'threads-batch': z.ZodOptional<z.ZodNumber>;
72
+ 'cpu-mask': z.ZodOptional<z.ZodString>;
73
+ 'cpu-mask-batch': z.ZodOptional<z.ZodString>;
74
+ 'override-tensor': z.ZodOptional<z.ZodString>;
75
+ 'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
76
+ 'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
77
+ 'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
78
+ 'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
79
+ auto: "auto";
80
+ on: "on";
81
+ off: "off";
82
+ }>>;
83
+ fit: z.ZodOptional<z.ZodBoolean>;
84
+ 'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
85
+ 'fit-ctx': z.ZodOptional<z.ZodNumber>;
61
86
  openclCacheDir: z.ZodOptional<z.ZodString>;
62
87
  reasoning_budget: z.ZodOptional<z.ZodNumber>;
63
88
  projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
@@ -113,7 +138,6 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
113
138
  export type LlmConfigInput = z.infer<typeof llmConfigBaseSchema>;
114
139
  export declare const LLM_CONFIG_DEFAULTS: {
115
140
  readonly ctx_size: 1024;
116
- readonly gpu_layers: 99;
117
141
  readonly device: "gpu";
118
142
  readonly system_prompt: "You are a helpful assistant.";
119
143
  readonly image_tile_mode: "sequential";
@@ -154,17 +178,42 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
154
178
  integrated: "integrated";
155
179
  dedicated: "dedicated";
156
180
  }>]>>;
181
+ 'rpc-servers': z.ZodOptional<z.ZodString>;
182
+ devices: z.ZodOptional<z.ZodString>;
157
183
  'split-mode': z.ZodOptional<z.ZodEnum<{
158
184
  none: "none";
159
185
  layer: "layer";
160
186
  tensor: "tensor";
161
187
  }>>;
162
188
  'flash-attn': z.ZodOptional<z.ZodEnum<{
189
+ auto: "auto";
163
190
  on: "on";
164
191
  off: "off";
165
- auto: "auto";
166
192
  }>>;
167
193
  'tensor-split': z.ZodOptional<z.ZodString>;
194
+ 'batch-size': z.ZodOptional<z.ZodNumber>;
195
+ 'ubatch-size': z.ZodOptional<z.ZodNumber>;
196
+ 'cpu-moe': z.ZodOptional<z.ZodBoolean>;
197
+ 'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
198
+ 'kv-offload': z.ZodOptional<z.ZodBoolean>;
199
+ 'image-max-tokens': z.ZodOptional<z.ZodNumber>;
200
+ 'image-min-tokens': z.ZodOptional<z.ZodNumber>;
201
+ threads: z.ZodOptional<z.ZodNumber>;
202
+ 'threads-batch': z.ZodOptional<z.ZodNumber>;
203
+ 'cpu-mask': z.ZodOptional<z.ZodString>;
204
+ 'cpu-mask-batch': z.ZodOptional<z.ZodString>;
205
+ 'override-tensor': z.ZodOptional<z.ZodString>;
206
+ 'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
207
+ 'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
208
+ 'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
209
+ 'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
210
+ auto: "auto";
211
+ on: "on";
212
+ off: "off";
213
+ }>>;
214
+ fit: z.ZodOptional<z.ZodBoolean>;
215
+ 'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
216
+ 'fit-ctx': z.ZodOptional<z.ZodNumber>;
168
217
  openclCacheDir: z.ZodOptional<z.ZodString>;
169
218
  reasoning_budget: z.ZodOptional<z.ZodNumber>;
170
219
  projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
@@ -222,7 +271,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
222
271
  top_p?: number | undefined;
223
272
  top_k?: number | undefined;
224
273
  seed?: number | undefined;
225
- gpu_layers: number | undefined;
274
+ gpu_layers?: number | undefined;
226
275
  lora?: string | undefined;
227
276
  device: string | undefined;
228
277
  predict?: number | undefined;
@@ -238,9 +287,30 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
238
287
  'cache-type-k'?: string | undefined;
239
288
  'cache-type-v'?: string | undefined;
240
289
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
290
+ 'rpc-servers'?: string | undefined;
291
+ devices?: string | undefined;
241
292
  'split-mode'?: "none" | "layer" | "tensor" | undefined;
242
- 'flash-attn'?: "on" | "off" | "auto" | undefined;
293
+ 'flash-attn'?: "auto" | "on" | "off" | undefined;
243
294
  'tensor-split'?: string | undefined;
295
+ 'batch-size'?: number | undefined;
296
+ 'ubatch-size'?: number | undefined;
297
+ 'cpu-moe'?: boolean | undefined;
298
+ 'n-cpu-moe'?: number | undefined;
299
+ 'kv-offload'?: boolean | undefined;
300
+ 'image-max-tokens'?: number | undefined;
301
+ 'image-min-tokens'?: number | undefined;
302
+ threads?: number | undefined;
303
+ 'threads-batch'?: number | undefined;
304
+ 'cpu-mask'?: string | undefined;
305
+ 'cpu-mask-batch'?: string | undefined;
306
+ 'override-tensor'?: string | undefined;
307
+ 'n-cpu-ffn'?: number | undefined;
308
+ 'moe-cache-mib'?: number | "auto" | undefined;
309
+ 'prefetch-weights'?: boolean | "auto" | undefined;
310
+ 'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
311
+ fit?: boolean | undefined;
312
+ 'fit-target'?: string | number | undefined;
313
+ 'fit-ctx'?: number | undefined;
244
314
  openclCacheDir?: string | undefined;
245
315
  reasoning_budget?: number | undefined;
246
316
  projectionModelSrc?: string | {
@@ -281,9 +351,30 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
281
351
  'cache-type-k'?: string | undefined;
282
352
  'cache-type-v'?: string | undefined;
283
353
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
354
+ 'rpc-servers'?: string | undefined;
355
+ devices?: string | undefined;
284
356
  'split-mode'?: "none" | "layer" | "tensor" | undefined;
285
- 'flash-attn'?: "on" | "off" | "auto" | undefined;
357
+ 'flash-attn'?: "auto" | "on" | "off" | undefined;
286
358
  'tensor-split'?: string | undefined;
359
+ 'batch-size'?: number | undefined;
360
+ 'ubatch-size'?: number | undefined;
361
+ 'cpu-moe'?: boolean | undefined;
362
+ 'n-cpu-moe'?: number | undefined;
363
+ 'kv-offload'?: boolean | undefined;
364
+ 'image-max-tokens'?: number | undefined;
365
+ 'image-min-tokens'?: number | undefined;
366
+ threads?: number | undefined;
367
+ 'threads-batch'?: number | undefined;
368
+ 'cpu-mask'?: string | undefined;
369
+ 'cpu-mask-batch'?: string | undefined;
370
+ 'override-tensor'?: string | undefined;
371
+ 'n-cpu-ffn'?: number | undefined;
372
+ 'moe-cache-mib'?: number | "auto" | undefined;
373
+ 'prefetch-weights'?: boolean | "auto" | undefined;
374
+ 'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
375
+ fit?: boolean | undefined;
376
+ 'fit-target'?: string | number | undefined;
377
+ 'fit-ctx'?: number | undefined;
287
378
  openclCacheDir?: string | undefined;
288
379
  reasoning_budget?: number | undefined;
289
380
  projectionModelSrc?: string | {
@@ -307,8 +398,8 @@ export type LlmConfig = z.infer<typeof llmConfigSchema>;
307
398
  export declare const embedConfigBaseSchema: z.ZodObject<{
308
399
  gpuLayers: z.ZodOptional<z.ZodNumber>;
309
400
  device: z.ZodOptional<z.ZodEnum<{
310
- gpu: "gpu";
311
401
  cpu: "cpu";
402
+ gpu: "gpu";
312
403
  }>>;
313
404
  batchSize: z.ZodOptional<z.ZodNumber>;
314
405
  pooling: z.ZodOptional<z.ZodEnum<{
@@ -324,9 +415,9 @@ export declare const embedConfigBaseSchema: z.ZodObject<{
324
415
  }>>;
325
416
  embdNormalize: z.ZodOptional<z.ZodNumber>;
326
417
  flashAttention: z.ZodOptional<z.ZodEnum<{
418
+ auto: "auto";
327
419
  on: "on";
328
420
  off: "off";
329
- auto: "auto";
330
421
  }>>;
331
422
  mainGpu: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
332
423
  integrated: "integrated";
@@ -354,8 +445,8 @@ export declare const EMBED_CONFIG_DEFAULTS: {
354
445
  export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
355
446
  gpuLayers: z.ZodOptional<z.ZodNumber>;
356
447
  device: z.ZodOptional<z.ZodEnum<{
357
- gpu: "gpu";
358
448
  cpu: "cpu";
449
+ gpu: "gpu";
359
450
  }>>;
360
451
  batchSize: z.ZodOptional<z.ZodNumber>;
361
452
  pooling: z.ZodOptional<z.ZodEnum<{
@@ -371,9 +462,9 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
371
462
  }>>;
372
463
  embdNormalize: z.ZodOptional<z.ZodNumber>;
373
464
  flashAttention: z.ZodOptional<z.ZodEnum<{
465
+ auto: "auto";
374
466
  on: "on";
375
467
  off: "off";
376
- auto: "auto";
377
468
  }>>;
378
469
  mainGpu: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
379
470
  integrated: "integrated";
@@ -393,12 +484,12 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
393
484
  openclCacheDir: z.ZodOptional<z.ZodString>;
394
485
  }, z.core.$strip>, z.ZodTransform<{
395
486
  gpuLayers: number | undefined;
396
- device: "gpu" | "cpu" | undefined;
487
+ device: "cpu" | "gpu" | undefined;
397
488
  batchSize: number | undefined;
398
489
  pooling?: "none" | "mean" | "cls" | "last" | "rank" | undefined;
399
490
  attention?: "causal" | "non-causal" | undefined;
400
491
  embdNormalize?: number | undefined;
401
- flashAttention?: "on" | "off" | "auto" | undefined;
492
+ flashAttention?: "auto" | "on" | "off" | undefined;
402
493
  mainGpu?: number | "integrated" | "dedicated" | undefined;
403
494
  splitMode?: "none" | "layer" | undefined;
404
495
  tensorSplit?: string | undefined;
@@ -406,12 +497,12 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
406
497
  openclCacheDir?: string | undefined;
407
498
  }, {
408
499
  gpuLayers?: number | undefined;
409
- device?: "gpu" | "cpu" | undefined;
500
+ device?: "cpu" | "gpu" | undefined;
410
501
  batchSize?: number | undefined;
411
502
  pooling?: "none" | "mean" | "cls" | "last" | "rank" | undefined;
412
503
  attention?: "causal" | "non-causal" | undefined;
413
504
  embdNormalize?: number | undefined;
414
- flashAttention?: "on" | "off" | "auto" | undefined;
505
+ flashAttention?: "auto" | "on" | "off" | undefined;
415
506
  mainGpu?: number | "integrated" | "dedicated" | undefined;
416
507
  splitMode?: "none" | "layer" | undefined;
417
508
  tensorSplit?: string | undefined;
@@ -37,7 +37,7 @@ export const llmConfigBaseSchema = z.object({
37
37
  gpu_layers: z
38
38
  .number()
39
39
  .optional()
40
- .describe('Number of model layers to offload to the GPU. Default 99 (offload all).'),
40
+ .describe('Number of model layers to offload to the GPU. Unset by default, which lets the runtime fit the placement to free device memory (offloading every layer when it fits). Setting it pins the layer count and disables that fit.'),
41
41
  lora: z
42
42
  .string()
43
43
  .optional()
@@ -103,10 +103,24 @@ export const llmConfigBaseSchema = z.object({
103
103
  .union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])])
104
104
  .optional()
105
105
  .describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
106
+ 'rpc-servers': z
107
+ .string()
108
+ .min(1)
109
+ .refine((value) => {
110
+ const endpoints = value.split(',').map((endpoint) => endpoint.trim());
111
+ return new Set(endpoints).size === endpoints.length;
112
+ }, 'RPC server endpoints must be unique')
113
+ .optional()
114
+ .describe('Comma-separated unique native RPC host:port endpoints in device registration order for this load. Requires compatible RPC addon builds and a trusted private network; native traffic is unencrypted and unauthenticated.'),
115
+ devices: z
116
+ .string()
117
+ .min(1)
118
+ .optional()
119
+ .describe("Explicit ordered native device names, for example RPC0,RPC1. RPC indices enumerate every device in this load's rpc-servers endpoint order, starting at RPC0 for each load. Pass exactly those endpoints in that order to getRpcDeviceMap, including on reused workers. tensor-split weights follow the selected devices order."),
106
120
  'split-mode': z
107
121
  .enum(['none', 'layer', 'tensor'])
108
122
  .optional()
109
- .describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across all visible GPUs; desktop-only, requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
123
+ .describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across selected local or RPC GPUs; requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
110
124
  'flash-attn': z
111
125
  .enum(['on', 'off', 'auto'])
112
126
  .optional()
@@ -115,6 +129,98 @@ export const llmConfigBaseSchema = z.object({
115
129
  .string()
116
130
  .optional()
117
131
  .describe("Proportions for distributing layers/rows across GPUs, e.g. `'1,1'` (equal) or `'3,1'` (75/25)."),
132
+ 'batch-size': z
133
+ .number()
134
+ .int()
135
+ .min(1)
136
+ .optional()
137
+ .describe('Logical maximum batch size, the tokens submitted per decode call. Default 2048.'),
138
+ 'ubatch-size': z
139
+ .number()
140
+ .int()
141
+ .min(1)
142
+ .optional()
143
+ .describe('Physical maximum batch size, the tokens computed in one pass. Default 512.'),
144
+ 'cpu-moe': z
145
+ .boolean()
146
+ .optional()
147
+ .describe('Keep all Mixture-of-Experts weights on the CPU, freeing accelerator memory for the rest of the model. Mixture-of-Experts models only. Default false.'),
148
+ 'n-cpu-moe': z
149
+ .number()
150
+ .int()
151
+ .min(0)
152
+ .optional()
153
+ .describe('Keep the Mixture-of-Experts weights of the first N layers on the CPU. Mixture-of-Experts models only, and a finer-grained alternative to `cpu-moe`.'),
154
+ 'kv-offload': z
155
+ .boolean()
156
+ .optional()
157
+ .describe('Whether to hold the KV cache in accelerator memory. Default true. Set false to keep it in system memory, which frees accelerator memory at the cost of decode speed.'),
158
+ 'image-max-tokens': z
159
+ .number()
160
+ .int()
161
+ .min(1)
162
+ .optional()
163
+ .describe('Upper bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
164
+ 'image-min-tokens': z
165
+ .number()
166
+ .int()
167
+ .min(1)
168
+ .optional()
169
+ .describe('Lower bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
170
+ threads: z
171
+ .number()
172
+ .int()
173
+ .optional()
174
+ .describe('CPU threads to use during generation. Unset uses one thread per physical core; a value of 0 or below uses every logical CPU, which is a different and usually larger count.'),
175
+ 'threads-batch': z
176
+ .number()
177
+ .int()
178
+ .optional()
179
+ .describe('CPU threads to use during batch and prompt processing. Setting it governs generation too and discards `threads` and `cpu-mask`; leave it unset for `threads` to apply to both.'),
180
+ 'cpu-mask': z
181
+ .string()
182
+ .optional()
183
+ .describe("CPU affinity mask as an arbitrarily long hex string, e.g. `'ff'`. Unset applies no affinity. Ignored when `threads-batch` is set."),
184
+ 'cpu-mask-batch': z
185
+ .string()
186
+ .optional()
187
+ .describe('CPU affinity mask for batch and prompt processing, in the same form as `cpu-mask`. Honoured only alongside `threads-batch`; without it `cpu-mask` takes over.'),
188
+ 'override-tensor': z
189
+ .string()
190
+ .optional()
191
+ .describe("Buffer type overrides for tensors matching a pattern, as comma-separated `<tensor name pattern>=<buffer type>` entries, e.g. `'blk\\.(1[0-9])\\.ffn_(up|down|gate)_exps=CPU'` to keep those experts in system memory."),
192
+ 'n-cpu-ffn': z
193
+ .number()
194
+ .int()
195
+ .min(0)
196
+ .optional()
197
+ .describe('Keep the dense FFN weights of the first N layers on the CPU.'),
198
+ 'moe-cache-mib': z
199
+ .union([z.number().int().min(0), z.literal('auto')])
200
+ .optional()
201
+ .describe("Persistent accelerator cache for Mixture-of-Experts weights, in MiB. Default `'auto'`, which sizes it during `fit`."),
202
+ 'prefetch-weights': z
203
+ .union([z.boolean(), z.literal('auto')])
204
+ .optional()
205
+ .describe("Overlap host-to-accelerator weight copies with compute. Default `'auto'`, which enables it for fitted dense models."),
206
+ 'tensor-read-lazy': z
207
+ .enum(['on', 'auto', 'off'])
208
+ .optional()
209
+ .describe("On-demand reading of large tensors such as per-layer embeddings, which requires mmap: `'on'` reads their rows from disk instead of keeping them resident, `'off'` always keeps them resident, `'auto'` (default) applies `'on'` above 4 GiB."),
210
+ fit: z
211
+ .boolean()
212
+ .optional()
213
+ .describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting `gpu_layers` pins the layer count and makes fit abort, leaving every argument unchanged. Incompatible with `split-mode: tensor`, which disables it.'),
214
+ 'fit-target': z
215
+ .union([z.number().int().min(0), z.string().regex(/^\d+(,\d+)*$/)])
216
+ .optional()
217
+ .describe("Free margin `fit` aims to leave per device, in MiB. A single value is broadcast across devices; a comma-separated list such as `'1024,512'` sets them individually. Default 1024."),
218
+ 'fit-ctx': z
219
+ .number()
220
+ .int()
221
+ .min(0)
222
+ .optional()
223
+ .describe('Smallest context size `fit` may settle on. Default 4096.'),
118
224
  openclCacheDir: z
119
225
  .string()
120
226
  .optional()
@@ -143,9 +249,13 @@ export const llmConfigBaseSchema = z.object({
143
249
  .describe('Run the multimodal projector (mmproj / vision encoder) on the GPU (multimodal models only). `true` forces GPU, `false` forces CPU. When unset, the backend is auto-selected per device class: GPU on desktop/iOS and Android Adreno 800+; CPU on every other Android GPU (Arm Mali, Adreno <800, and undetectable Adreno tiers), with the LLM layers still on the GPU. Only honoured when the model itself runs on a GPU backend — ignored with a warning on CPU.')
144
250
  });
145
251
  // Default values - typed as partial of the config
252
+ //
253
+ // gpu_layers carries no default: any value reads as user intent to qvac-fabric,
254
+ // which then aborts its fit instead of placing layers to free device memory.
255
+ // Absent emits no `--gpu-layers`, so the addon keeps fabric's own -1 — already
256
+ // "every layer" whenever the fit does not run.
146
257
  export const LLM_CONFIG_DEFAULTS = {
147
258
  ctx_size: 1024,
148
- gpu_layers: 99,
149
259
  device: 'gpu',
150
260
  system_prompt: 'You are a helpful assistant.',
151
261
  image_tile_mode: 'sequential'