@qvac/inference 0.19.1 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +57 -17
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +69 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +191 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +67 -0
- package/dist/schemas/assess-model-fit.js +56 -2
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +816 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +186 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +88 -8
- package/dist/schemas/llamacpp-config.js +108 -8
- package/dist/schemas/load-model.d.ts +2906 -206
- package/dist/schemas/model-config-schemas.d.ts +251 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
package/dist/schemas/common.js
CHANGED
|
@@ -15,6 +15,7 @@ import { loggingStreamRequestSchema, loggingStreamResponseSchema } from './loggi
|
|
|
15
15
|
import { ttsRequestSchema, ttsResponseSchema, textToSpeechStreamRequestSchema, textToSpeechStreamResponseSchema } from './text-to-speech.js';
|
|
16
16
|
import { errorResponseSchema } from './error.js';
|
|
17
17
|
import { ragRequestSchema, ragResponseSchema, ragProgressUpdateSchema } from './rag.js';
|
|
18
|
+
import { vectorIndexRequestSchema, vectorIndexResponseSchema } from './vector-index.js';
|
|
18
19
|
import { getModelInfoRequestSchema, getModelInfoResponseSchema, getLoadedModelInfoRequestSchema, getLoadedModelInfoResponseSchema } from './model-info.js';
|
|
19
20
|
import { getSystemResourcesRequestSchema, getSystemResourcesResponseSchema } from './system-resources.js';
|
|
20
21
|
import { assessModelFitRequestSchema, assessModelFitResponseSchema } from './assess-model-fit.js';
|
|
@@ -25,9 +26,11 @@ import { pluginInvokeRequestSchema, pluginInvokeResponseSchema, pluginInvokeStre
|
|
|
25
26
|
import { modelRegistryListRequestSchema, modelRegistryListResponseSchema, modelRegistrySearchRequestSchema, modelRegistrySearchResponseSchema, modelRegistryGetModelRequestSchema, modelRegistryGetModelResponseSchema } from './registry.js';
|
|
26
27
|
import { suspendRequestSchema, suspendResponseSchema, resumeRequestSchema, resumeResponseSchema, stateRequestSchema, stateResponseSchema } from './lifecycle.js';
|
|
27
28
|
import { classifyRequestSchema, classifyResponseSchema } from './classification.js';
|
|
28
|
-
import { audioGenStreamRequestSchema, audioGenStreamResponseSchema } from './audio-gen.js';
|
|
29
|
+
import { audioEditStreamRequestSchema, audioEditStreamResponseSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, audioUnderstandRequestSchema, audioUnderstandResponseSchema } from './audio-gen.js';
|
|
29
30
|
export const requestSchema = z.union([
|
|
30
31
|
audioGenStreamRequestSchema,
|
|
32
|
+
audioEditStreamRequestSchema,
|
|
33
|
+
audioUnderstandRequestSchema,
|
|
31
34
|
heartbeatRequestSchema,
|
|
32
35
|
loadModelRequestSchema,
|
|
33
36
|
downloadAssetRequestSchema,
|
|
@@ -46,6 +49,7 @@ export const requestSchema = z.union([
|
|
|
46
49
|
textToSpeechStreamRequestSchema,
|
|
47
50
|
cancelRequestSchema,
|
|
48
51
|
ragRequestSchema,
|
|
52
|
+
vectorIndexRequestSchema,
|
|
49
53
|
deleteCacheRequestSchema,
|
|
50
54
|
getModelInfoRequestSchema,
|
|
51
55
|
getLoadedModelInfoRequestSchema,
|
|
@@ -70,6 +74,8 @@ export const requestSchema = z.union([
|
|
|
70
74
|
]);
|
|
71
75
|
export const responseSchema = z.discriminatedUnion('type', [
|
|
72
76
|
audioGenStreamResponseSchema,
|
|
77
|
+
audioEditStreamResponseSchema,
|
|
78
|
+
audioUnderstandResponseSchema,
|
|
73
79
|
heartbeatResponseSchema,
|
|
74
80
|
loadModelResponseSchema,
|
|
75
81
|
downloadAssetResponseSchema,
|
|
@@ -91,6 +97,7 @@ export const responseSchema = z.discriminatedUnion('type', [
|
|
|
91
97
|
errorResponseSchema,
|
|
92
98
|
ragResponseSchema,
|
|
93
99
|
ragProgressUpdateSchema,
|
|
100
|
+
vectorIndexResponseSchema,
|
|
94
101
|
deleteCacheResponseSchema,
|
|
95
102
|
getModelInfoResponseSchema,
|
|
96
103
|
getLoadedModelInfoResponseSchema,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { type Tool, type ToolCallWithCall } from './tools.js';
|
|
2
|
+
import { type Tool, type ToolCallError, type ToolCallWithCall } from './tools.js';
|
|
3
3
|
import type { ToolDialect } from './completion-stream.js';
|
|
4
4
|
export declare const completionStatsSchema: z.ZodObject<{
|
|
5
5
|
timeToFirstToken: z.ZodOptional<z.ZodNumber>;
|
|
@@ -10,6 +10,7 @@ export declare const completionStatsSchema: z.ZodObject<{
|
|
|
10
10
|
generatedTokens: z.ZodOptional<z.ZodNumber>;
|
|
11
11
|
emittedTokens: z.ZodOptional<z.ZodNumber>;
|
|
12
12
|
avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
|
|
13
|
+
toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
|
|
13
14
|
backendDevice: z.ZodOptional<z.ZodEnum<{
|
|
14
15
|
gpu: "gpu";
|
|
15
16
|
cpu: "cpu";
|
|
@@ -67,6 +68,7 @@ export declare const statsEventSchema: z.ZodObject<{
|
|
|
67
68
|
generatedTokens: z.ZodOptional<z.ZodNumber>;
|
|
68
69
|
emittedTokens: z.ZodOptional<z.ZodNumber>;
|
|
69
70
|
avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
|
|
71
|
+
toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
|
|
70
72
|
backendDevice: z.ZodOptional<z.ZodEnum<{
|
|
71
73
|
gpu: "gpu";
|
|
72
74
|
cpu: "cpu";
|
|
@@ -150,6 +152,7 @@ export declare const completionEventSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
150
152
|
generatedTokens: z.ZodOptional<z.ZodNumber>;
|
|
151
153
|
emittedTokens: z.ZodOptional<z.ZodNumber>;
|
|
152
154
|
avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
|
|
155
|
+
toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
|
|
153
156
|
backendDevice: z.ZodOptional<z.ZodEnum<{
|
|
154
157
|
gpu: "gpu";
|
|
155
158
|
cpu: "cpu";
|
|
@@ -192,6 +195,13 @@ export type CompletionFinal = {
|
|
|
192
195
|
contentText: string;
|
|
193
196
|
thinkingText?: string;
|
|
194
197
|
toolCalls: ToolCallWithCall[];
|
|
198
|
+
/**
|
|
199
|
+
* Tool-call regions the model emitted that could not be turned into a
|
|
200
|
+
* `toolCalls` entry — malformed markup, arguments failing the tool's
|
|
201
|
+
* schema, or a call to an undeclared tool. Omitted when there were none.
|
|
202
|
+
* Lets a caller tell "the model made a mistake" from "no tool was called".
|
|
203
|
+
*/
|
|
204
|
+
toolErrors?: ToolCallError[];
|
|
195
205
|
stats?: CompletionStats;
|
|
196
206
|
stopReason?: StopReason;
|
|
197
207
|
raw: {
|
|
@@ -11,6 +11,9 @@ export const completionStatsSchema = z.object({
|
|
|
11
11
|
// Non-empty addon stream pieces — prefer for usage reporting when present.
|
|
12
12
|
emittedTokens: z.number().optional(),
|
|
13
13
|
avgConcurrentSeq: z.number().optional(),
|
|
14
|
+
// Prompt renders that provably left the request's tool definitions out.
|
|
15
|
+
// Non-zero means the model never saw them; 0 is not proof it saw them all.
|
|
16
|
+
toolDefinitionsDropped: z.number().optional(),
|
|
14
17
|
backendDevice: z.enum(['cpu', 'gpu']).optional()
|
|
15
18
|
});
|
|
16
19
|
export const seqSchema = z.number().int().nonnegative();
|
|
@@ -37,7 +37,22 @@ export declare const generationParamsSchema: z.ZodObject<{
|
|
|
37
37
|
repeat_penalty: z.ZodOptional<z.ZodNumber>;
|
|
38
38
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
39
39
|
remove_thinking_from_context: z.ZodOptional<z.ZodBoolean>;
|
|
40
|
+
tool_choice: z.ZodOptional<z.ZodString>;
|
|
40
41
|
}, z.core.$strict>;
|
|
42
|
+
/**
|
|
43
|
+
* Whether a `tool_choice` demands a tool call (`required` or a named tool),
|
|
44
|
+
* as opposed to `auto` / `none`, which only shape how the model may call one.
|
|
45
|
+
*/
|
|
46
|
+
export declare function toolChoiceDemandsCall(toolChoice: string | undefined): boolean;
|
|
47
|
+
export declare function refineToolChoiceMatchesTools(data: {
|
|
48
|
+
tools?: {
|
|
49
|
+
type: 'function';
|
|
50
|
+
name: string;
|
|
51
|
+
}[] | undefined;
|
|
52
|
+
generationParams?: {
|
|
53
|
+
tool_choice?: string | undefined;
|
|
54
|
+
} | undefined;
|
|
55
|
+
}, ctx: z.RefinementCtx): void;
|
|
41
56
|
export declare const responseFormatSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
42
57
|
type: z.ZodLiteral<"text">;
|
|
43
58
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -105,6 +120,7 @@ export declare const completionClientParamsSchema: z.ZodObject<{
|
|
|
105
120
|
repeat_penalty: z.ZodOptional<z.ZodNumber>;
|
|
106
121
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
107
122
|
remove_thinking_from_context: z.ZodOptional<z.ZodBoolean>;
|
|
123
|
+
tool_choice: z.ZodOptional<z.ZodString>;
|
|
108
124
|
}, z.core.$strict>>;
|
|
109
125
|
captureThinking: z.ZodOptional<z.ZodBoolean>;
|
|
110
126
|
emitRawDeltas: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -175,6 +191,7 @@ export declare const completionStreamRequestSchema: z.ZodObject<{
|
|
|
175
191
|
repeat_penalty: z.ZodOptional<z.ZodNumber>;
|
|
176
192
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
177
193
|
remove_thinking_from_context: z.ZodOptional<z.ZodBoolean>;
|
|
194
|
+
tool_choice: z.ZodOptional<z.ZodString>;
|
|
178
195
|
}, z.core.$strict>>;
|
|
179
196
|
captureThinking: z.ZodOptional<z.ZodBoolean>;
|
|
180
197
|
emitRawDeltas: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -251,6 +268,7 @@ export declare const completionStreamResponseSchema: z.ZodObject<{
|
|
|
251
268
|
generatedTokens: z.ZodOptional<z.ZodNumber>;
|
|
252
269
|
emittedTokens: z.ZodOptional<z.ZodNumber>;
|
|
253
270
|
avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
|
|
271
|
+
toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
|
|
254
272
|
backendDevice: z.ZodOptional<z.ZodEnum<{
|
|
255
273
|
gpu: "gpu";
|
|
256
274
|
cpu: "cpu";
|
|
@@ -323,6 +341,7 @@ export declare const completionOrchestrateRequestSchema: z.ZodObject<{
|
|
|
323
341
|
repeat_penalty: z.ZodOptional<z.ZodNumber>;
|
|
324
342
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
325
343
|
remove_thinking_from_context: z.ZodOptional<z.ZodBoolean>;
|
|
344
|
+
tool_choice: z.ZodOptional<z.ZodString>;
|
|
326
345
|
}, z.core.$strict>>;
|
|
327
346
|
captureThinking: z.ZodOptional<z.ZodBoolean>;
|
|
328
347
|
emitRawDeltas: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -413,6 +432,7 @@ export declare const completionOrchestrateResponseSchema: z.ZodObject<{
|
|
|
413
432
|
generatedTokens: z.ZodOptional<z.ZodNumber>;
|
|
414
433
|
emittedTokens: z.ZodOptional<z.ZodNumber>;
|
|
415
434
|
avgConcurrentSeq: z.ZodOptional<z.ZodNumber>;
|
|
435
|
+
toolDefinitionsDropped: z.ZodOptional<z.ZodNumber>;
|
|
416
436
|
backendDevice: z.ZodOptional<z.ZodEnum<{
|
|
417
437
|
gpu: "gpu";
|
|
418
438
|
cpu: "cpu";
|
|
@@ -64,9 +64,42 @@ export const generationParamsSchema = z
|
|
|
64
64
|
remove_thinking_from_context: z
|
|
65
65
|
.boolean()
|
|
66
66
|
.optional()
|
|
67
|
-
.describe('When the model emits a reasoning block during generation (e.g. `<think>...</think>` for the Qwen3 family, `<|channel>thought ... <channel|>` for Gemma 4), drop those tokens from the KV cache at end-of-generation so subsequent turns do not accumulate reasoning history. Defaults to `false`, except the Qwen3 reasoning family (Qwen3, Qwen3.5, Qwen3.6, including MoE variants), which defaults to `true`. No-op for models without a recognised reasoning channel. Supported on recurrent / hybrid-SSM models (e.g. Qwen3.5) via a state snapshot and replay when the reasoning close marker is a single token; on such a model with a multi-token close marker, enabling this fails with an error.')
|
|
67
|
+
.describe('When the model emits a reasoning block during generation (e.g. `<think>...</think>` for the Qwen3 family, `<|channel>thought ... <channel|>` for Gemma 4), drop those tokens from the KV cache at end-of-generation so subsequent turns do not accumulate reasoning history. Defaults to `false`, except the Qwen3 reasoning family (Qwen3, Qwen3.5, Qwen3.6, including MoE variants), which defaults to `true`. No-op for models without a recognised reasoning channel. Supported on recurrent / hybrid-SSM models (e.g. Qwen3.5) via a state snapshot and replay when the reasoning close marker is a single token; on such a model with a multi-token close marker, enabling this fails with an error.'),
|
|
68
|
+
tool_choice: z
|
|
69
|
+
.string()
|
|
70
|
+
.min(1)
|
|
71
|
+
.optional()
|
|
72
|
+
.describe('Controls tool calling for a request that declares `tools`, in the OpenAI style. `"auto"` (default) lets the model decide and constrains output to the tool-call grammar only once it starts a call; `"required"` forces a tool call; `"none"` leaves the tool definitions in the prompt but disables the tool-call grammar; any other value names one declared tool and forces a call to it. `"required"` and a tool name are rejected when the request declares no tools, and fail the request rather than answering in prose when the model cannot honour them. Only honoured by llama.cpp-backed models; other backends ignore it.')
|
|
68
73
|
})
|
|
69
74
|
.strict();
|
|
75
|
+
const TOOL_CHOICE_MODES = new Set(['auto', 'none', 'required']);
|
|
76
|
+
/**
|
|
77
|
+
* Whether a `tool_choice` demands a tool call (`required` or a named tool),
|
|
78
|
+
* as opposed to `auto` / `none`, which only shape how the model may call one.
|
|
79
|
+
*/
|
|
80
|
+
export function toolChoiceDemandsCall(toolChoice) {
|
|
81
|
+
return toolChoice !== undefined && toolChoice !== 'auto' && toolChoice !== 'none';
|
|
82
|
+
}
|
|
83
|
+
export function refineToolChoiceMatchesTools(data, ctx) {
|
|
84
|
+
const toolChoice = data.generationParams?.tool_choice;
|
|
85
|
+
if (toolChoice === undefined || TOOL_CHOICE_MODES.has(toolChoice)) {
|
|
86
|
+
if (toolChoice === 'required' && !(data.tools && data.tools.length > 0)) {
|
|
87
|
+
ctx.addIssue({
|
|
88
|
+
code: 'custom',
|
|
89
|
+
message: 'generationParams.tool_choice "required" needs at least one tool.',
|
|
90
|
+
path: ['generationParams', 'tool_choice']
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (!data.tools?.some((tool) => tool.name === toolChoice)) {
|
|
96
|
+
ctx.addIssue({
|
|
97
|
+
code: 'custom',
|
|
98
|
+
message: `generationParams.tool_choice names "${toolChoice}", which is not one of the declared tools.`,
|
|
99
|
+
path: ['generationParams', 'tool_choice']
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
}
|
|
70
103
|
const jsonSchemaObjectSchema = z.record(z.string(), z.unknown());
|
|
71
104
|
export const responseFormatSchema = z.discriminatedUnion('type', [
|
|
72
105
|
z.object({ type: z.literal('text') }).strict(),
|
|
@@ -154,12 +187,15 @@ function refineNoToolsWithStructuredOutput(data, ctx) {
|
|
|
154
187
|
});
|
|
155
188
|
}
|
|
156
189
|
}
|
|
157
|
-
export const completionClientParamsSchema = completionClientParamsBaseSchema
|
|
190
|
+
export const completionClientParamsSchema = completionClientParamsBaseSchema
|
|
191
|
+
.superRefine(refineNoToolsWithStructuredOutput)
|
|
192
|
+
.superRefine(refineToolChoiceMatchesTools);
|
|
158
193
|
export const completionStreamRequestSchema = completionClientParamsBaseSchema
|
|
159
194
|
.extend({
|
|
160
195
|
type: z.literal('completionStream')
|
|
161
196
|
})
|
|
162
|
-
.superRefine(refineNoToolsWithStructuredOutput)
|
|
197
|
+
.superRefine(refineNoToolsWithStructuredOutput)
|
|
198
|
+
.superRefine(refineToolChoiceMatchesTools);
|
|
163
199
|
export const completionStreamResponseSchema = z
|
|
164
200
|
.object({
|
|
165
201
|
type: z.literal('completionStream'),
|
|
@@ -184,7 +220,8 @@ export const completionOrchestrateRequestSchema = completionClientParamsBaseSche
|
|
|
184
220
|
'pass; a turn that requests tool calls consumes callbacks and starts ' +
|
|
185
221
|
'the next turn. Defaults to 8.')
|
|
186
222
|
})
|
|
187
|
-
.superRefine(refineNoToolsWithStructuredOutput)
|
|
223
|
+
.superRefine(refineNoToolsWithStructuredOutput)
|
|
224
|
+
.superRefine(refineToolChoiceMatchesTools);
|
|
188
225
|
/**
|
|
189
226
|
* Downstream frame of the orchestrated completion duplex stream. Exactly one
|
|
190
227
|
* payload field is set per frame:
|