@qvac/inference 0.19.1 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/NOTICE +57 -17
  2. package/dist/api/assess-model-fit.d.ts +9 -4
  3. package/dist/api/assess-model-fit.js +9 -4
  4. package/dist/api/audio-gen.d.ts +42 -1
  5. package/dist/api/audio-gen.js +137 -51
  6. package/dist/api/index.d.ts +2 -1
  7. package/dist/api/index.js +2 -1
  8. package/dist/api/text-to-speech.d.ts +31 -4
  9. package/dist/api/text-to-speech.js +117 -21
  10. package/dist/api/vector-index.d.ts +142 -0
  11. package/dist/api/vector-index.js +168 -0
  12. package/dist/api/world.d.ts +2 -2
  13. package/dist/errors/index.d.ts +12 -0
  14. package/dist/errors/index.js +21 -0
  15. package/dist/handlers/assess-model-fit.d.ts +30 -6
  16. package/dist/handlers/assess-model-fit.js +68 -6
  17. package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
  18. package/dist/handlers/load-model/registry-download-utils.js +2 -1
  19. package/dist/handlers/vector-index.d.ts +2 -0
  20. package/dist/handlers/vector-index.js +40 -0
  21. package/dist/index.d.ts +1 -1
  22. package/dist/index.js +1 -1
  23. package/dist/models/registry/models.d.ts +351 -12
  24. package/dist/models/registry/models.js +2271 -1957
  25. package/dist/models/registry/resource-profiles.js +135 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  30. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  31. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  32. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  33. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  34. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  35. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  36. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  37. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  38. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  39. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  40. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  41. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  42. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  43. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  44. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  45. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  46. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  47. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  48. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  49. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  50. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  51. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
  52. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
  53. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  54. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  55. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +69 -4
  56. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  57. package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
  58. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  59. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  60. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  61. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  62. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  63. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  64. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  65. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  66. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  67. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  68. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  69. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  70. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  71. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  72. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  73. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  74. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  75. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  76. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  77. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  78. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  79. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  80. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  81. package/dist/plugins/index.d.ts +1 -1
  82. package/dist/plugins/index.js +1 -1
  83. package/dist/plugins/ops/load-model.js +22 -2
  84. package/dist/plugins/ops/translate.js +58 -12
  85. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  86. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  87. package/dist/plugins/registry.d.ts +15 -0
  88. package/dist/plugins/registry.js +133 -15
  89. package/dist/plugins/turbovec-backend.d.ts +17 -0
  90. package/dist/plugins/turbovec-backend.js +99 -0
  91. package/dist/registry.js +4 -0
  92. package/dist/resources/model-fit/assess.d.ts +10 -1
  93. package/dist/resources/model-fit/assess.js +43 -0
  94. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
  95. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
  96. package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
  97. package/dist/resources/model-fit/fit-stub/index.js +2 -0
  98. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
  99. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
  100. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  101. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  102. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  103. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +191 -0
  104. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  105. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  106. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  107. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  108. package/dist/runtime/env.d.ts +2 -0
  109. package/dist/runtime/env.js +9 -2
  110. package/dist/runtime/lifecycle.js +2 -0
  111. package/dist/runtime/model-registry.d.ts +3 -1
  112. package/dist/runtime/model-registry.js +2 -1
  113. package/dist/runtime/request-context.js +17 -0
  114. package/dist/runtime/vector-index-backend.d.ts +25 -0
  115. package/dist/runtime/vector-index-backend.js +1 -0
  116. package/dist/runtime/vector-index-registry.d.ts +76 -0
  117. package/dist/runtime/vector-index-registry.js +161 -0
  118. package/dist/schemas/assess-model-fit.d.ts +67 -0
  119. package/dist/schemas/assess-model-fit.js +56 -2
  120. package/dist/schemas/audio-gen.d.ts +479 -4
  121. package/dist/schemas/audio-gen.js +376 -17
  122. package/dist/schemas/batch-completion-stream.d.ts +7 -0
  123. package/dist/schemas/batch-completion-stream.js +5 -3
  124. package/dist/schemas/common.d.ts +816 -15
  125. package/dist/schemas/common.js +8 -1
  126. package/dist/schemas/completion-event.d.ts +11 -1
  127. package/dist/schemas/completion-event.js +3 -0
  128. package/dist/schemas/completion-stream.d.ts +20 -0
  129. package/dist/schemas/completion-stream.js +41 -4
  130. package/dist/schemas/config.d.ts +186 -30
  131. package/dist/schemas/errors.d.ts +4 -0
  132. package/dist/schemas/errors.js +22 -0
  133. package/dist/schemas/index.d.ts +1 -0
  134. package/dist/schemas/index.js +1 -0
  135. package/dist/schemas/llamacpp-config.d.ts +88 -8
  136. package/dist/schemas/llamacpp-config.js +108 -8
  137. package/dist/schemas/load-model.d.ts +2906 -206
  138. package/dist/schemas/model-config-schemas.d.ts +251 -5
  139. package/dist/schemas/plugin.d.ts +10 -0
  140. package/dist/schemas/sdcpp-config.d.ts +221 -4
  141. package/dist/schemas/sdcpp-config.js +128 -22
  142. package/dist/schemas/text-to-speech.d.ts +780 -19
  143. package/dist/schemas/text-to-speech.js +298 -44
  144. package/dist/schemas/transcription-config.js +7 -4
  145. package/dist/schemas/vector-index.d.ts +194 -0
  146. package/dist/schemas/vector-index.js +185 -0
  147. package/dist/utils/addon-responses.d.ts +29 -3
  148. package/dist/utils/aggregate-events.d.ts +3 -1
  149. package/dist/utils/aggregate-events.js +7 -1
  150. package/dist/utils/tts-stats.d.ts +15 -11
  151. package/dist/utils/tts-stats.js +33 -11
  152. package/package.json +29 -26
@@ -50,7 +50,6 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
50
50
  'split-mode': z.ZodOptional<z.ZodEnum<{
51
51
  none: "none";
52
52
  layer: "layer";
53
- row: "row";
54
53
  tensor: "tensor";
55
54
  }>>;
56
55
  'flash-attn': z.ZodOptional<z.ZodEnum<{
@@ -59,6 +58,29 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
59
58
  auto: "auto";
60
59
  }>>;
61
60
  'tensor-split': z.ZodOptional<z.ZodString>;
61
+ 'batch-size': z.ZodOptional<z.ZodNumber>;
62
+ 'ubatch-size': z.ZodOptional<z.ZodNumber>;
63
+ 'cpu-moe': z.ZodOptional<z.ZodBoolean>;
64
+ 'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
65
+ 'kv-offload': z.ZodOptional<z.ZodBoolean>;
66
+ 'image-max-tokens': z.ZodOptional<z.ZodNumber>;
67
+ 'image-min-tokens': z.ZodOptional<z.ZodNumber>;
68
+ threads: z.ZodOptional<z.ZodNumber>;
69
+ 'threads-batch': z.ZodOptional<z.ZodNumber>;
70
+ 'cpu-mask': z.ZodOptional<z.ZodString>;
71
+ 'cpu-mask-batch': z.ZodOptional<z.ZodString>;
72
+ 'override-tensor': z.ZodOptional<z.ZodString>;
73
+ 'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
74
+ 'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
75
+ 'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
76
+ 'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
77
+ on: "on";
78
+ off: "off";
79
+ auto: "auto";
80
+ }>>;
81
+ fit: z.ZodOptional<z.ZodBoolean>;
82
+ 'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
83
+ 'fit-ctx': z.ZodOptional<z.ZodNumber>;
62
84
  openclCacheDir: z.ZodOptional<z.ZodString>;
63
85
  reasoning_budget: z.ZodOptional<z.ZodNumber>;
64
86
  projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
@@ -158,7 +180,6 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
158
180
  'split-mode': z.ZodOptional<z.ZodEnum<{
159
181
  none: "none";
160
182
  layer: "layer";
161
- row: "row";
162
183
  tensor: "tensor";
163
184
  }>>;
164
185
  'flash-attn': z.ZodOptional<z.ZodEnum<{
@@ -167,6 +188,29 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
167
188
  auto: "auto";
168
189
  }>>;
169
190
  'tensor-split': z.ZodOptional<z.ZodString>;
191
+ 'batch-size': z.ZodOptional<z.ZodNumber>;
192
+ 'ubatch-size': z.ZodOptional<z.ZodNumber>;
193
+ 'cpu-moe': z.ZodOptional<z.ZodBoolean>;
194
+ 'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
195
+ 'kv-offload': z.ZodOptional<z.ZodBoolean>;
196
+ 'image-max-tokens': z.ZodOptional<z.ZodNumber>;
197
+ 'image-min-tokens': z.ZodOptional<z.ZodNumber>;
198
+ threads: z.ZodOptional<z.ZodNumber>;
199
+ 'threads-batch': z.ZodOptional<z.ZodNumber>;
200
+ 'cpu-mask': z.ZodOptional<z.ZodString>;
201
+ 'cpu-mask-batch': z.ZodOptional<z.ZodString>;
202
+ 'override-tensor': z.ZodOptional<z.ZodString>;
203
+ 'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
204
+ 'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
205
+ 'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
206
+ 'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
207
+ on: "on";
208
+ off: "off";
209
+ auto: "auto";
210
+ }>>;
211
+ fit: z.ZodOptional<z.ZodBoolean>;
212
+ 'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
213
+ 'fit-ctx': z.ZodOptional<z.ZodNumber>;
170
214
  openclCacheDir: z.ZodOptional<z.ZodString>;
171
215
  reasoning_budget: z.ZodOptional<z.ZodNumber>;
172
216
  projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
@@ -240,9 +284,28 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
240
284
  'cache-type-k'?: string | undefined;
241
285
  'cache-type-v'?: string | undefined;
242
286
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
243
- 'split-mode'?: "none" | "layer" | "row" | "tensor" | undefined;
287
+ 'split-mode'?: "none" | "layer" | "tensor" | undefined;
244
288
  'flash-attn'?: "on" | "off" | "auto" | undefined;
245
289
  'tensor-split'?: string | undefined;
290
+ 'batch-size'?: number | undefined;
291
+ 'ubatch-size'?: number | undefined;
292
+ 'cpu-moe'?: boolean | undefined;
293
+ 'n-cpu-moe'?: number | undefined;
294
+ 'kv-offload'?: boolean | undefined;
295
+ 'image-max-tokens'?: number | undefined;
296
+ 'image-min-tokens'?: number | undefined;
297
+ threads?: number | undefined;
298
+ 'threads-batch'?: number | undefined;
299
+ 'cpu-mask'?: string | undefined;
300
+ 'cpu-mask-batch'?: string | undefined;
301
+ 'override-tensor'?: string | undefined;
302
+ 'n-cpu-ffn'?: number | undefined;
303
+ 'moe-cache-mib'?: number | "auto" | undefined;
304
+ 'prefetch-weights'?: boolean | "auto" | undefined;
305
+ 'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
306
+ fit?: boolean | undefined;
307
+ 'fit-target'?: string | number | undefined;
308
+ 'fit-ctx'?: number | undefined;
246
309
  openclCacheDir?: string | undefined;
247
310
  reasoning_budget?: number | undefined;
248
311
  projectionModelSrc?: string | {
@@ -283,9 +346,28 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
283
346
  'cache-type-k'?: string | undefined;
284
347
  'cache-type-v'?: string | undefined;
285
348
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
286
- 'split-mode'?: "none" | "layer" | "row" | "tensor" | undefined;
349
+ 'split-mode'?: "none" | "layer" | "tensor" | undefined;
287
350
  'flash-attn'?: "on" | "off" | "auto" | undefined;
288
351
  'tensor-split'?: string | undefined;
352
+ 'batch-size'?: number | undefined;
353
+ 'ubatch-size'?: number | undefined;
354
+ 'cpu-moe'?: boolean | undefined;
355
+ 'n-cpu-moe'?: number | undefined;
356
+ 'kv-offload'?: boolean | undefined;
357
+ 'image-max-tokens'?: number | undefined;
358
+ 'image-min-tokens'?: number | undefined;
359
+ threads?: number | undefined;
360
+ 'threads-batch'?: number | undefined;
361
+ 'cpu-mask'?: string | undefined;
362
+ 'cpu-mask-batch'?: string | undefined;
363
+ 'override-tensor'?: string | undefined;
364
+ 'n-cpu-ffn'?: number | undefined;
365
+ 'moe-cache-mib'?: number | "auto" | undefined;
366
+ 'prefetch-weights'?: boolean | "auto" | undefined;
367
+ 'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
368
+ fit?: boolean | undefined;
369
+ 'fit-target'?: string | number | undefined;
370
+ 'fit-ctx'?: number | undefined;
289
371
  openclCacheDir?: string | undefined;
290
372
  reasoning_budget?: number | undefined;
291
373
  projectionModelSrc?: string | {
@@ -337,7 +419,6 @@ export declare const embedConfigBaseSchema: z.ZodObject<{
337
419
  splitMode: z.ZodOptional<z.ZodEnum<{
338
420
  none: "none";
339
421
  layer: "layer";
340
- row: "row";
341
422
  }>>;
342
423
  tensorSplit: z.ZodOptional<z.ZodString>;
343
424
  verbosity: z.ZodOptional<z.ZodEnum<{
@@ -385,7 +466,6 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
385
466
  splitMode: z.ZodOptional<z.ZodEnum<{
386
467
  none: "none";
387
468
  layer: "layer";
388
- row: "row";
389
469
  }>>;
390
470
  tensorSplit: z.ZodOptional<z.ZodString>;
391
471
  verbosity: z.ZodOptional<z.ZodEnum<{
@@ -404,7 +484,7 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
404
484
  embdNormalize?: number | undefined;
405
485
  flashAttention?: "on" | "off" | "auto" | undefined;
406
486
  mainGpu?: number | "integrated" | "dedicated" | undefined;
407
- splitMode?: "none" | "layer" | "row" | undefined;
487
+ splitMode?: "none" | "layer" | undefined;
408
488
  tensorSplit?: string | undefined;
409
489
  verbosity?: 0 | 3 | 2 | 1 | undefined;
410
490
  openclCacheDir?: string | undefined;
@@ -417,7 +497,7 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
417
497
  embdNormalize?: number | undefined;
418
498
  flashAttention?: "on" | "off" | "auto" | undefined;
419
499
  mainGpu?: number | "integrated" | "dedicated" | undefined;
420
- splitMode?: "none" | "layer" | "row" | undefined;
500
+ splitMode?: "none" | "layer" | undefined;
421
501
  tensorSplit?: string | undefined;
422
502
  verbosity?: 0 | 3 | 2 | 1 | undefined;
423
503
  openclCacheDir?: string | undefined;
@@ -104,9 +104,9 @@ export const llmConfigBaseSchema = z.object({
104
104
  .optional()
105
105
  .describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
106
106
  'split-mode': z
107
- .enum(['none', 'layer', 'row', 'tensor'])
107
+ .enum(['none', 'layer', 'tensor'])
108
108
  .optional()
109
- .describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), `'row'` (legacy; degrades to `'layer'`), or `'tensor'` (EXPERIMENTAL tensor parallelism across all visible GPUs; desktop-only, requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
109
+ .describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across all visible GPUs; desktop-only, requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
110
110
  'flash-attn': z
111
111
  .enum(['on', 'off', 'auto'])
112
112
  .optional()
@@ -115,6 +115,98 @@ export const llmConfigBaseSchema = z.object({
115
115
  .string()
116
116
  .optional()
117
117
  .describe("Proportions for distributing layers/rows across GPUs, e.g. `'1,1'` (equal) or `'3,1'` (75/25)."),
118
+ 'batch-size': z
119
+ .number()
120
+ .int()
121
+ .min(1)
122
+ .optional()
123
+ .describe('Logical maximum batch size, the tokens submitted per decode call. Default 2048.'),
124
+ 'ubatch-size': z
125
+ .number()
126
+ .int()
127
+ .min(1)
128
+ .optional()
129
+ .describe('Physical maximum batch size, the tokens computed in one pass. Default 512.'),
130
+ 'cpu-moe': z
131
+ .boolean()
132
+ .optional()
133
+ .describe('Keep all Mixture-of-Experts weights on the CPU, freeing accelerator memory for the rest of the model. Mixture-of-Experts models only. Default false.'),
134
+ 'n-cpu-moe': z
135
+ .number()
136
+ .int()
137
+ .min(0)
138
+ .optional()
139
+ .describe('Keep the Mixture-of-Experts weights of the first N layers on the CPU. Mixture-of-Experts models only, and a finer-grained alternative to `cpu-moe`.'),
140
+ 'kv-offload': z
141
+ .boolean()
142
+ .optional()
143
+ .describe('Whether to hold the KV cache in accelerator memory. Default true. Set false to keep it in system memory, which frees accelerator memory at the cost of decode speed.'),
144
+ 'image-max-tokens': z
145
+ .number()
146
+ .int()
147
+ .min(1)
148
+ .optional()
149
+ .describe('Upper bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
150
+ 'image-min-tokens': z
151
+ .number()
152
+ .int()
153
+ .min(1)
154
+ .optional()
155
+ .describe('Lower bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
156
+ threads: z
157
+ .number()
158
+ .int()
159
+ .optional()
160
+ .describe('CPU threads to use during generation. Unset uses one thread per physical core; a value of 0 or below uses every logical CPU, which is a different and usually larger count.'),
161
+ 'threads-batch': z
162
+ .number()
163
+ .int()
164
+ .optional()
165
+ .describe('CPU threads to use during batch and prompt processing. Setting it governs generation too and discards `threads` and `cpu-mask`; leave it unset for `threads` to apply to both.'),
166
+ 'cpu-mask': z
167
+ .string()
168
+ .optional()
169
+ .describe("CPU affinity mask as an arbitrarily long hex string, e.g. `'ff'`. Unset applies no affinity. Ignored when `threads-batch` is set."),
170
+ 'cpu-mask-batch': z
171
+ .string()
172
+ .optional()
173
+ .describe('CPU affinity mask for batch and prompt processing, in the same form as `cpu-mask`. Honoured only alongside `threads-batch`; without it `cpu-mask` takes over.'),
174
+ 'override-tensor': z
175
+ .string()
176
+ .optional()
177
+ .describe("Buffer type overrides for tensors matching a pattern, as comma-separated `<tensor name pattern>=<buffer type>` entries, e.g. `'blk\\.(1[0-9])\\.ffn_(up|down|gate)_exps=CPU'` to keep those experts in system memory."),
178
+ 'n-cpu-ffn': z
179
+ .number()
180
+ .int()
181
+ .min(0)
182
+ .optional()
183
+ .describe('Keep the dense FFN weights of the first N layers on the CPU.'),
184
+ 'moe-cache-mib': z
185
+ .union([z.number().int().min(0), z.literal('auto')])
186
+ .optional()
187
+ .describe("Persistent accelerator cache for Mixture-of-Experts weights, in MiB. Default `'auto'`, which sizes it during `fit`."),
188
+ 'prefetch-weights': z
189
+ .union([z.boolean(), z.literal('auto')])
190
+ .optional()
191
+ .describe("Overlap host-to-accelerator weight copies with compute. Default `'auto'`, which enables it for fitted dense models."),
192
+ 'tensor-read-lazy': z
193
+ .enum(['on', 'auto', 'off'])
194
+ .optional()
195
+ .describe("On-demand reading of large tensors such as per-layer embeddings, which requires mmap: `'on'` reads their rows from disk instead of keeping them resident, `'off'` always keeps them resident, `'auto'` (default) applies `'on'` above 4 GiB."),
196
+ fit: z
197
+ .boolean()
198
+ .optional()
199
+ .describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting this field releases the `gpu_layers` default so fit can choose it, which offloads every layer rather than 99 if fit then gives up; setting `gpu_layers` alongside it pins the value and makes fit abort, leaving every argument unchanged. Incompatible with `split-mode: tensor`, which disables it.'),
200
+ 'fit-target': z
201
+ .union([z.number().int().min(0), z.string().regex(/^\d+(,\d+)*$/)])
202
+ .optional()
203
+ .describe("Free margin `fit` aims to leave per device, in MiB. A single value is broadcast across devices; a comma-separated list such as `'1024,512'` sets them individually. Default 1024."),
204
+ 'fit-ctx': z
205
+ .number()
206
+ .int()
207
+ .min(0)
208
+ .optional()
209
+ .describe('Smallest context size `fit` may settle on. Default 4096.'),
118
210
  openclCacheDir: z
119
211
  .string()
120
212
  .optional()
@@ -150,11 +242,19 @@ export const LLM_CONFIG_DEFAULTS = {
150
242
  system_prompt: 'You are a helpful assistant.',
151
243
  image_tile_mode: 'sequential'
152
244
  };
245
+ /**
246
+ * `llama_model_default_params().n_gpu_layers`. Fabric's fit aborts and rolls
247
+ * back every argument it had computed when `n_gpu_layers` holds any other
248
+ * value, so a caller opting into `fit` gets this in place of the default above.
249
+ */
250
+ const LLAMA_GPU_LAYERS_DEFAULT = -1;
153
251
  // Full schema - applies defaults via transform (no duplication)
154
- export const llmConfigSchema = llmConfigBaseSchema.transform((data) => ({
155
- ...LLM_CONFIG_DEFAULTS,
156
- ...data
157
- }));
252
+ export const llmConfigSchema = llmConfigBaseSchema.transform((data) => {
253
+ const merged = { ...LLM_CONFIG_DEFAULTS, ...data };
254
+ return data.fit !== undefined && data.gpu_layers === undefined
255
+ ? { ...merged, gpu_layers: LLAMA_GPU_LAYERS_DEFAULT }
256
+ : merged;
257
+ });
158
258
  // Base schema - validates types, all fields optional (for input validation)
159
259
  export const embedConfigBaseSchema = z.object({
160
260
  gpuLayers: z
@@ -194,9 +294,9 @@ export const embedConfigBaseSchema = z.object({
194
294
  .optional()
195
295
  .describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
196
296
  splitMode: z
197
- .enum(['none', 'layer', 'row'])
297
+ .enum(['none', 'layer'])
198
298
  .optional()
199
- .describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'row'` (tensor parallelism)."),
299
+ .describe("How to split the model across GPUs: `'none'` (default, single GPU) or `'layer'` (pipeline parallelism)."),
200
300
  tensorSplit: z
201
301
  .string()
202
302
  .optional()