@qvac/inference 0.19.1 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/NOTICE +57 -17
  2. package/dist/api/assess-model-fit.d.ts +9 -4
  3. package/dist/api/assess-model-fit.js +9 -4
  4. package/dist/api/audio-gen.d.ts +42 -1
  5. package/dist/api/audio-gen.js +137 -51
  6. package/dist/api/index.d.ts +2 -1
  7. package/dist/api/index.js +2 -1
  8. package/dist/api/text-to-speech.d.ts +31 -4
  9. package/dist/api/text-to-speech.js +117 -21
  10. package/dist/api/vector-index.d.ts +142 -0
  11. package/dist/api/vector-index.js +168 -0
  12. package/dist/api/world.d.ts +2 -2
  13. package/dist/errors/index.d.ts +12 -0
  14. package/dist/errors/index.js +21 -0
  15. package/dist/handlers/assess-model-fit.d.ts +30 -6
  16. package/dist/handlers/assess-model-fit.js +68 -6
  17. package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
  18. package/dist/handlers/load-model/registry-download-utils.js +2 -1
  19. package/dist/handlers/vector-index.d.ts +2 -0
  20. package/dist/handlers/vector-index.js +40 -0
  21. package/dist/index.d.ts +1 -1
  22. package/dist/index.js +1 -1
  23. package/dist/models/registry/models.d.ts +351 -12
  24. package/dist/models/registry/models.js +2271 -1957
  25. package/dist/models/registry/resource-profiles.js +135 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  30. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  31. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  32. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  33. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  34. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  35. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  36. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  37. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  38. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  39. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  40. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  41. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  42. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  43. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  44. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  45. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  46. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  47. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  48. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  49. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  50. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  51. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
  52. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
  53. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  54. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  55. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +69 -4
  56. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  57. package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
  58. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  59. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  60. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  61. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  62. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  63. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  64. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  65. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  66. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  67. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  68. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  69. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  70. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  71. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  72. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  73. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  74. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  75. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  76. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  77. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  78. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  79. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  80. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  81. package/dist/plugins/index.d.ts +1 -1
  82. package/dist/plugins/index.js +1 -1
  83. package/dist/plugins/ops/load-model.js +22 -2
  84. package/dist/plugins/ops/translate.js +58 -12
  85. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  86. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  87. package/dist/plugins/registry.d.ts +15 -0
  88. package/dist/plugins/registry.js +133 -15
  89. package/dist/plugins/turbovec-backend.d.ts +17 -0
  90. package/dist/plugins/turbovec-backend.js +99 -0
  91. package/dist/registry.js +4 -0
  92. package/dist/resources/model-fit/assess.d.ts +10 -1
  93. package/dist/resources/model-fit/assess.js +43 -0
  94. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
  95. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
  96. package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
  97. package/dist/resources/model-fit/fit-stub/index.js +2 -0
  98. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
  99. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
  100. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  101. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  102. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  103. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +191 -0
  104. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  105. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  106. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  107. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  108. package/dist/runtime/env.d.ts +2 -0
  109. package/dist/runtime/env.js +9 -2
  110. package/dist/runtime/lifecycle.js +2 -0
  111. package/dist/runtime/model-registry.d.ts +3 -1
  112. package/dist/runtime/model-registry.js +2 -1
  113. package/dist/runtime/request-context.js +17 -0
  114. package/dist/runtime/vector-index-backend.d.ts +25 -0
  115. package/dist/runtime/vector-index-backend.js +1 -0
  116. package/dist/runtime/vector-index-registry.d.ts +76 -0
  117. package/dist/runtime/vector-index-registry.js +161 -0
  118. package/dist/schemas/assess-model-fit.d.ts +67 -0
  119. package/dist/schemas/assess-model-fit.js +56 -2
  120. package/dist/schemas/audio-gen.d.ts +479 -4
  121. package/dist/schemas/audio-gen.js +376 -17
  122. package/dist/schemas/batch-completion-stream.d.ts +7 -0
  123. package/dist/schemas/batch-completion-stream.js +5 -3
  124. package/dist/schemas/common.d.ts +816 -15
  125. package/dist/schemas/common.js +8 -1
  126. package/dist/schemas/completion-event.d.ts +11 -1
  127. package/dist/schemas/completion-event.js +3 -0
  128. package/dist/schemas/completion-stream.d.ts +20 -0
  129. package/dist/schemas/completion-stream.js +41 -4
  130. package/dist/schemas/config.d.ts +186 -30
  131. package/dist/schemas/errors.d.ts +4 -0
  132. package/dist/schemas/errors.js +22 -0
  133. package/dist/schemas/index.d.ts +1 -0
  134. package/dist/schemas/index.js +1 -0
  135. package/dist/schemas/llamacpp-config.d.ts +88 -8
  136. package/dist/schemas/llamacpp-config.js +108 -8
  137. package/dist/schemas/load-model.d.ts +2906 -206
  138. package/dist/schemas/model-config-schemas.d.ts +251 -5
  139. package/dist/schemas/plugin.d.ts +10 -0
  140. package/dist/schemas/sdcpp-config.d.ts +221 -4
  141. package/dist/schemas/sdcpp-config.js +128 -22
  142. package/dist/schemas/text-to-speech.d.ts +780 -19
  143. package/dist/schemas/text-to-speech.js +298 -44
  144. package/dist/schemas/transcription-config.js +7 -4
  145. package/dist/schemas/vector-index.d.ts +194 -0
  146. package/dist/schemas/vector-index.js +185 -0
  147. package/dist/utils/addon-responses.d.ts +29 -3
  148. package/dist/utils/aggregate-events.d.ts +3 -1
  149. package/dist/utils/aggregate-events.js +7 -1
  150. package/dist/utils/tts-stats.d.ts +15 -11
  151. package/dist/utils/tts-stats.js +33 -11
  152. package/package.json +29 -26
@@ -15,6 +15,7 @@ import { loggingStreamRequestSchema, loggingStreamResponseSchema } from './loggi
15
15
  import { ttsRequestSchema, ttsResponseSchema, textToSpeechStreamRequestSchema, textToSpeechStreamResponseSchema } from './text-to-speech.js';
16
16
  import { errorResponseSchema } from './error.js';
17
17
  import { ragRequestSchema, ragResponseSchema, ragProgressUpdateSchema } from './rag.js';
18
+ import { vectorIndexRequestSchema, vectorIndexResponseSchema } from './vector-index.js';
18
19
  import { getModelInfoRequestSchema, getModelInfoResponseSchema, getLoadedModelInfoRequestSchema, getLoadedModelInfoResponseSchema } from './model-info.js';
19
20
  import { getSystemResourcesRequestSchema, getSystemResourcesResponseSchema } from './system-resources.js';
20
21
  import { assessModelFitRequestSchema, assessModelFitResponseSchema } from './assess-model-fit.js';
@@ -25,9 +26,11 @@ import { pluginInvokeRequestSchema, pluginInvokeResponseSchema, pluginInvokeStre
25
26
  import { modelRegistryListRequestSchema, modelRegistryListResponseSchema, modelRegistrySearchRequestSchema, modelRegistrySearchResponseSchema, modelRegistryGetModelRequestSchema, modelRegistryGetModelResponseSchema } from './registry.js';
26
27
  import { suspendRequestSchema, suspendResponseSchema, resumeRequestSchema, resumeResponseSchema, stateRequestSchema, stateResponseSchema } from './lifecycle.js';
27
28
  import { classifyRequestSchema, classifyResponseSchema } from './classification.js';
28
- import { audioGenStreamRequestSchema, audioGenStreamResponseSchema } from './audio-gen.js';
29
+ import { audioEditStreamRequestSchema, audioEditStreamResponseSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, audioUnderstandRequestSchema, audioUnderstandResponseSchema } from './audio-gen.js';
29
30
  export const requestSchema = z.union([
30
31
  audioGenStreamRequestSchema,
32
+ audioEditStreamRequestSchema,
33
+ audioUnderstandRequestSchema,
31
34
  heartbeatRequestSchema,
32
35
  loadModelRequestSchema,
33
36
  downloadAssetRequestSchema,
@@ -46,6 +49,7 @@ export const requestSchema = z.union([
46
49
  textToSpeechStreamRequestSchema,
47
50
  cancelRequestSchema,
48
51
  ragRequestSchema,
52
+ vectorIndexRequestSchema,
49
53
  deleteCacheRequestSchema,
50
54
  getModelInfoRequestSchema,
51
55
  getLoadedModelInfoRequestSchema,
@@ -70,6 +74,8 @@ export const requestSchema = z.union([
70
74
  ]);
71
75
  export const responseSchema = z.discriminatedUnion('type', [
72
76
  audioGenStreamResponseSchema,
77
+ audioEditStreamResponseSchema,
78
+ audioUnderstandResponseSchema,
73
79
  heartbeatResponseSchema,
74
80
  loadModelResponseSchema,
75
81
  downloadAssetResponseSchema,
@@ -91,6 +97,7 @@ export const responseSchema = z.discriminatedUnion('type', [
91
97
  errorResponseSchema,
92
98
  ragResponseSchema,
93
99
  ragProgressUpdateSchema,
100
+ vectorIndexResponseSchema,
94
101
  deleteCacheResponseSchema,
95
102
  getModelInfoResponseSchema,
96
103
  getLoadedModelInfoResponseSchema,
@@ -1,5 +1,5 @@
1
1
  import { z } from 'zod';
2
- import { type Tool, type ToolCallWithCall } from './tools.js';
2
+ import { type Tool, type ToolCallError, type ToolCallWithCall } from './tools.js';
3
3
  import type { ToolDialect } from './completion-stream.js';
4
4
  export declare const completionStatsSchema: z.ZodObject<{
5
5
  timeToFirstToken: z.ZodOptional<z.ZodNumber>;
@@ -10,6 +10,7 @@ export declare const completionStatsSchema: z.ZodObject<{
10
10
  generatedTokens: z.ZodOptional<z.ZodNumber>;
11
11
  emittedTokens: z.ZodOptional<z.ZodNumber>;
12
12
  avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
13
+ toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
13
14
  backendDevice: z.ZodOptional<z.ZodEnum<{
14
15
  gpu: "gpu";
15
16
  cpu: "cpu";
@@ -67,6 +68,7 @@ export declare const statsEventSchema: z.ZodObject<{
67
68
  generatedTokens: z.ZodOptional<z.ZodNumber>;
68
69
  emittedTokens: z.ZodOptional<z.ZodNumber>;
69
70
  avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
71
+ toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
70
72
  backendDevice: z.ZodOptional<z.ZodEnum<{
71
73
  gpu: "gpu";
72
74
  cpu: "cpu";
@@ -150,6 +152,7 @@ export declare const completionEventSchema: z.ZodUnion<readonly [z.ZodObject<{
150
152
  generatedTokens: z.ZodOptional<z.ZodNumber>;
151
153
  emittedTokens: z.ZodOptional<z.ZodNumber>;
152
154
  avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
155
+ toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
153
156
  backendDevice: z.ZodOptional<z.ZodEnum<{
154
157
  gpu: "gpu";
155
158
  cpu: "cpu";
@@ -192,6 +195,13 @@ export type CompletionFinal = {
192
195
  contentText: string;
193
196
  thinkingText?: string;
194
197
  toolCalls: ToolCallWithCall[];
198
+ /**
199
+ * Tool-call regions the model emitted that could not be turned into a
200
+ * `toolCalls` entry — malformed markup, arguments failing the tool's
201
+ * schema, or a call to an undeclared tool. Omitted when there were none.
202
+ * Lets a caller tell "the model made a mistake" from "no tool was called".
203
+ */
204
+ toolErrors?: ToolCallError[];
195
205
  stats?: CompletionStats;
196
206
  stopReason?: StopReason;
197
207
  raw: {
@@ -11,6 +11,9 @@ export const completionStatsSchema = z.object({
11
11
  // Non-empty addon stream pieces — prefer for usage reporting when present.
12
12
  emittedTokens: z.number().optional(),
13
13
  avgConcurrentSeq: z.number().optional(),
14
+ // Prompt renders that provably left the request's tool definitions out.
15
+ // Non-zero means the model never saw them; 0 is not proof it saw them all.
16
+ toolDefinitionsDropped: z.number().optional(),
14
17
  backendDevice: z.enum(['cpu', 'gpu']).optional()
15
18
  });
16
19
  export const seqSchema = z.number().int().nonnegative();
@@ -37,7 +37,22 @@ export declare const generationParamsSchema: z.ZodObject<{
37
37
  repeat_penalty: z.ZodOptional<z.ZodNumber>;
38
38
  reasoning_budget: z.ZodOptional<z.ZodNumber>;
39
39
  remove_thinking_from_context: z.ZodOptional<z.ZodBoolean>;
40
+ tool_choice: z.ZodOptional<z.ZodString>;
40
41
  }, z.core.$strict>;
42
+ /**
43
+ * Whether a `tool_choice` demands a tool call (`required` or a named tool),
44
+ * as opposed to `auto` / `none`, which only shape how the model may call one.
45
+ */
46
+ export declare function toolChoiceDemandsCall(toolChoice: string | undefined): boolean;
47
+ export declare function refineToolChoiceMatchesTools(data: {
48
+ tools?: {
49
+ type: 'function';
50
+ name: string;
51
+ }[] | undefined;
52
+ generationParams?: {
53
+ tool_choice?: string | undefined;
54
+ } | undefined;
55
+ }, ctx: z.RefinementCtx): void;
41
56
  export declare const responseFormatSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
42
57
  type: z.ZodLiteral<"text">;
43
58
  }, z.core.$strict>, z.ZodObject<{
@@ -105,6 +120,7 @@ export declare const completionClientParamsSchema: z.ZodObject<{
105
120
  repeat_penalty: z.ZodOptional<z.ZodNumber>;
106
121
  reasoning_budget: z.ZodOptional<z.ZodNumber>;
107
122
  remove_thinking_from_context: z.ZodOptional<z.ZodBoolean>;
123
+ tool_choice: z.ZodOptional<z.ZodString>;
108
124
  }, z.core.$strict>>;
109
125
  captureThinking: z.ZodOptional<z.ZodBoolean>;
110
126
  emitRawDeltas: z.ZodOptional<z.ZodBoolean>;
@@ -175,6 +191,7 @@ export declare const completionStreamRequestSchema: z.ZodObject<{
175
191
  repeat_penalty: z.ZodOptional<z.ZodNumber>;
176
192
  reasoning_budget: z.ZodOptional<z.ZodNumber>;
177
193
  remove_thinking_from_context: z.ZodOptional<z.ZodBoolean>;
194
+ tool_choice: z.ZodOptional<z.ZodString>;
178
195
  }, z.core.$strict>>;
179
196
  captureThinking: z.ZodOptional<z.ZodBoolean>;
180
197
  emitRawDeltas: z.ZodOptional<z.ZodBoolean>;
@@ -251,6 +268,7 @@ export declare const completionStreamResponseSchema: z.ZodObject<{
251
268
  generatedTokens: z.ZodOptional<z.ZodNumber>;
252
269
  emittedTokens: z.ZodOptional<z.ZodNumber>;
253
270
  avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
271
+ toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
254
272
  backendDevice: z.ZodOptional<z.ZodEnum<{
255
273
  gpu: "gpu";
256
274
  cpu: "cpu";
@@ -323,6 +341,7 @@ export declare const completionOrchestrateRequestSchema: z.ZodObject<{
323
341
  repeat_penalty: z.ZodOptional<z.ZodNumber>;
324
342
  reasoning_budget: z.ZodOptional<z.ZodNumber>;
325
343
  remove_thinking_from_context: z.ZodOptional<z.ZodBoolean>;
344
+ tool_choice: z.ZodOptional<z.ZodString>;
326
345
  }, z.core.$strict>>;
327
346
  captureThinking: z.ZodOptional<z.ZodBoolean>;
328
347
  emitRawDeltas: z.ZodOptional<z.ZodBoolean>;
@@ -413,6 +432,7 @@ export declare const completionOrchestrateResponseSchema: z.ZodObject<{
413
432
  generatedTokens: z.ZodOptional<z.ZodNumber>;
414
433
  emittedTokens: z.ZodOptional<z.ZodNumber>;
415
434
  avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
435
+ toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
416
436
  backendDevice: z.ZodOptional<z.ZodEnum<{
417
437
  gpu: "gpu";
418
438
  cpu: "cpu";
@@ -64,9 +64,42 @@ export const generationParamsSchema = z
64
64
  remove_thinking_from_context: z
65
65
  .boolean()
66
66
  .optional()
67
- .describe('When the model emits a reasoning block during generation (e.g. `<think>...</think>` for the Qwen3 family, `<|channel>thought ... <channel|>` for Gemma 4), drop those tokens from the KV cache at end-of-generation so subsequent turns do not accumulate reasoning history. Defaults to `false`, except the Qwen3 reasoning family (Qwen3, Qwen3.5, Qwen3.6, including MoE variants), which defaults to `true`. No-op for models without a recognised reasoning channel. Supported on recurrent / hybrid-SSM models (e.g. Qwen3.5) via a state snapshot and replay when the reasoning close marker is a single token; on such a model with a multi-token close marker, enabling this fails with an error.')
67
+ .describe('When the model emits a reasoning block during generation (e.g. `<think>...</think>` for the Qwen3 family, `<|channel>thought ... <channel|>` for Gemma 4), drop those tokens from the KV cache at end-of-generation so subsequent turns do not accumulate reasoning history. Defaults to `false`, except the Qwen3 reasoning family (Qwen3, Qwen3.5, Qwen3.6, including MoE variants), which defaults to `true`. No-op for models without a recognised reasoning channel. Supported on recurrent / hybrid-SSM models (e.g. Qwen3.5) via a state snapshot and replay when the reasoning close marker is a single token; on such a model with a multi-token close marker, enabling this fails with an error.'),
68
+ tool_choice: z
69
+ .string()
70
+ .min(1)
71
+ .optional()
72
+ .describe('Controls tool calling for a request that declares `tools`, in the OpenAI style. `"auto"` (default) lets the model decide and constrains output to the tool-call grammar only once it starts a call; `"required"` forces a tool call; `"none"` leaves the tool definitions in the prompt but disables the tool-call grammar; any other value names one declared tool and forces a call to it. `"required"` and a tool name are rejected when the request declares no tools, and fail the request rather than answering in prose when the model cannot honour them. Only honoured by llama.cpp-backed models; other backends ignore it.')
68
73
  })
69
74
  .strict();
75
+ const TOOL_CHOICE_MODES = new Set(['auto', 'none', 'required']);
76
+ /**
77
+ * Whether a `tool_choice` demands a tool call (`required` or a named tool),
78
+ * as opposed to `auto` / `none`, which only shape how the model may call one.
79
+ */
80
+ export function toolChoiceDemandsCall(toolChoice) {
81
+ return toolChoice !== undefined && toolChoice !== 'auto' && toolChoice !== 'none';
82
+ }
83
+ export function refineToolChoiceMatchesTools(data, ctx) {
84
+ const toolChoice = data.generationParams?.tool_choice;
85
+ if (toolChoice === undefined || TOOL_CHOICE_MODES.has(toolChoice)) {
86
+ if (toolChoice === 'required' && !(data.tools && data.tools.length > 0)) {
87
+ ctx.addIssue({
88
+ code: 'custom',
89
+ message: 'generationParams.tool_choice "required" needs at least one tool.',
90
+ path: ['generationParams', 'tool_choice']
91
+ });
92
+ }
93
+ return;
94
+ }
95
+ if (!data.tools?.some((tool) => tool.name === toolChoice)) {
96
+ ctx.addIssue({
97
+ code: 'custom',
98
+ message: `generationParams.tool_choice names "${toolChoice}", which is not one of the declared tools.`,
99
+ path: ['generationParams', 'tool_choice']
100
+ });
101
+ }
102
+ }
70
103
  const jsonSchemaObjectSchema = z.record(z.string(), z.unknown());
71
104
  export const responseFormatSchema = z.discriminatedUnion('type', [
72
105
  z.object({ type: z.literal('text') }).strict(),
@@ -154,12 +187,15 @@ function refineNoToolsWithStructuredOutput(data, ctx) {
154
187
  });
155
188
  }
156
189
  }
157
- export const completionClientParamsSchema = completionClientParamsBaseSchema.superRefine(refineNoToolsWithStructuredOutput);
190
+ export const completionClientParamsSchema = completionClientParamsBaseSchema
191
+ .superRefine(refineNoToolsWithStructuredOutput)
192
+ .superRefine(refineToolChoiceMatchesTools);
158
193
  export const completionStreamRequestSchema = completionClientParamsBaseSchema
159
194
  .extend({
160
195
  type: z.literal('completionStream')
161
196
  })
162
- .superRefine(refineNoToolsWithStructuredOutput);
197
+ .superRefine(refineNoToolsWithStructuredOutput)
198
+ .superRefine(refineToolChoiceMatchesTools);
163
199
  export const completionStreamResponseSchema = z
164
200
  .object({
165
201
  type: z.literal('completionStream'),
@@ -184,7 +220,8 @@ export const completionOrchestrateRequestSchema = completionClientParamsBaseSche
184
220
  'pass; a turn that requests tool calls consumes callbacks and starts ' +
185
221
  'the next turn. Defaults to 8.')
186
222
  })
187
- .superRefine(refineNoToolsWithStructuredOutput);
223
+ .superRefine(refineNoToolsWithStructuredOutput)
224
+ .superRefine(refineToolChoiceMatchesTools);
188
225
  /**
189
226
  * Downstream frame of the orchestrated completion duplex stream. Exactly one
190
227
  * payload field is set per frame: