@qvac/inference 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +43 -11
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +44 -0
- package/dist/schemas/assess-model-fit.js +40 -0
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +749 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2503 -139
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +28 -25
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import bciAddonLogging from '@qvac/bci-whispercpp/addonLogging';
|
|
2
1
|
import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
|
|
3
2
|
import { resolveBciConfig } from './resolve-config.js';
|
|
4
3
|
export declare const bciPlugin: {
|
|
@@ -167,7 +166,11 @@ export declare const bciPlugin: {
|
|
|
167
166
|
}, import("zod/v4/core").$strip>>;
|
|
168
167
|
};
|
|
169
168
|
logging: {
|
|
170
|
-
module:
|
|
169
|
+
module: () => Promise<{
|
|
170
|
+
default: typeof import("@qvac/bci-whispercpp/addonLogging");
|
|
171
|
+
setLogger: (callback: (priority: number, message: string) => void) => void;
|
|
172
|
+
releaseLogger: () => void;
|
|
173
|
+
}>;
|
|
171
174
|
namespace: "bci-whispercpp-transcription";
|
|
172
175
|
};
|
|
173
176
|
};
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import bciAddonLogging from '@qvac/bci-whispercpp/addonLogging';
|
|
2
1
|
import BCIWhispercpp from '@qvac/bci-whispercpp';
|
|
3
2
|
import { definePlugin, defineHandler, defineDuplexHandler, bciTranscribeRequestSchema, bciTranscribeResponseSchema, bciTranscribeStreamRequestSchema, bciTranscribeStreamResponseSchema, ModelType, bciConfigSchema, ADDON_BCI } from '../../../schemas/index.js';
|
|
4
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
@@ -105,7 +104,7 @@ export const bciPlugin = definePlugin({
|
|
|
105
104
|
})
|
|
106
105
|
},
|
|
107
106
|
logging: {
|
|
108
|
-
module:
|
|
107
|
+
module: () => import('@qvac/bci-whispercpp/addonLogging'),
|
|
109
108
|
namespace: ModelType.bciWhispercppTranscription
|
|
110
109
|
}
|
|
111
110
|
});
|
|
@@ -96,7 +96,7 @@ export declare const ocrPlugin: {
|
|
|
96
96
|
}, import("zod/v4/core").$strip>>;
|
|
97
97
|
};
|
|
98
98
|
logging: {
|
|
99
|
-
module: import("@qvac/ocr-ggml/addonLogging")
|
|
99
|
+
module: () => Promise<typeof import("@qvac/ocr-ggml/addonLogging")>;
|
|
100
100
|
namespace: "ggml-ocr";
|
|
101
101
|
};
|
|
102
102
|
};
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { definePlugin, defineHandler, ocrStreamRequestSchema, ocrStreamResponseSchema, ModelType, ocrConfigSchema, ADDON_OCR } from '../../../schemas/index.js';
|
|
2
2
|
import { ModelLoadFailedError } from '../../../errors/index.js';
|
|
3
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
4
|
-
import ocrAddonLogging from '@qvac/ocr-ggml/addonLogging';
|
|
5
4
|
import { OcrGgml } from '@qvac/ocr-ggml';
|
|
6
5
|
import { ocr } from './ops/ocr-stream.js';
|
|
7
6
|
import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
|
|
@@ -106,7 +105,7 @@ export const ocrPlugin = definePlugin({
|
|
|
106
105
|
})
|
|
107
106
|
},
|
|
108
107
|
logging: {
|
|
109
|
-
module:
|
|
108
|
+
module: () => import('@qvac/ocr-ggml/addonLogging'),
|
|
110
109
|
namespace: ModelType.ggmlOcr
|
|
111
110
|
}
|
|
112
111
|
});
|
|
@@ -3,7 +3,7 @@ import { getEngineLogger } from '../../../../logging/index.js';
|
|
|
3
3
|
import { nowMs } from '../../../../profiling/index.js';
|
|
4
4
|
import { buildStreamResult } from '../../../../profiling/model-execution.js';
|
|
5
5
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
6
|
-
import { transformMessages } from './completion-stream.js';
|
|
6
|
+
import { seedConfiguredSystemPrompt, transformMessages } from './completion-stream.js';
|
|
7
7
|
import { normalizeCompletionStats } from './completion-stats.js';
|
|
8
8
|
import { prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
9
9
|
const logger = getEngineLogger();
|
|
@@ -24,9 +24,10 @@ function mergeGenerationParams(generationParams, responseFormat) {
|
|
|
24
24
|
}
|
|
25
25
|
function renderPromptHistory(prompt, options) {
|
|
26
26
|
const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0 ? prompt.tools : undefined;
|
|
27
|
-
|
|
27
|
+
const history = seedConfiguredSystemPrompt(prompt.history, options.modelConfig);
|
|
28
|
+
let historyWithTools = history;
|
|
28
29
|
if (tools) {
|
|
29
|
-
historyWithTools = prependToolsToHistory(
|
|
30
|
+
historyWithTools = prependToolsToHistory(history, tools);
|
|
30
31
|
}
|
|
31
32
|
// Uses the same attachment expansion as single completion: each
|
|
32
33
|
// attachment becomes an addon `type: "media"` message before the text turn.
|
|
@@ -57,7 +58,8 @@ export async function* batchCompletion(params, opts) {
|
|
|
57
58
|
const model = getModel(modelId);
|
|
58
59
|
const modelConfig = getModelConfig(modelId);
|
|
59
60
|
const renderOptions = {
|
|
60
|
-
toolsEnabled: modelConfig.tools === true
|
|
61
|
+
toolsEnabled: modelConfig.tools === true,
|
|
62
|
+
modelConfig
|
|
61
63
|
};
|
|
62
64
|
// Cancel via the batch response, mirroring completion(): the addon routes
|
|
63
65
|
// response.cancel() to cancelJob for this batch group only, so a concurrent
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
export declare function logCacheStatus(cacheKey: string, isReusing: boolean): void;
|
|
2
|
-
export declare function
|
|
3
|
-
export declare function logMessagesToAddon(messages: unknown[], phase?: 'CACHE_INIT' | 'PROMPT_SEND' | 'NO_CACHE'): void;
|
|
2
|
+
export declare function logMessagesToAddon(messages: unknown[], phase?: 'PROMPT_SEND' | 'NO_CACHE'): void;
|
|
4
3
|
export declare function logCacheDisabled(): void;
|
|
5
4
|
export declare function logCacheSave(sessionPath: string): void;
|
|
6
5
|
export declare function logCacheSaveError(sessionPath: string, err: unknown): void;
|
|
@@ -17,10 +17,6 @@ export function logCacheStatus(cacheKey, isReusing) {
|
|
|
17
17
|
const status = isReusing ? 'REUSING' : 'CREATING';
|
|
18
18
|
logger.debug(`[kv-cache] [${cacheKey}] ${status} cache`);
|
|
19
19
|
}
|
|
20
|
-
export function logCacheInit(cacheKey, systemPrompt) {
|
|
21
|
-
const promptLen = systemPrompt.length;
|
|
22
|
-
logger.debug(`[kv-cache] [${cacheKey}] Initializing cache (prompt: ${promptLen} chars)`);
|
|
23
|
-
}
|
|
24
20
|
export function logMessagesToAddon(messages, phase = 'PROMPT_SEND') {
|
|
25
21
|
const typedMessages = messages;
|
|
26
22
|
logger.debug(`[kv-cache] [${phase}] Sending ${typedMessages.length} msg(s): [${formatMessages(typedMessages)}]`);
|
|
@@ -9,6 +9,7 @@ export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
|
|
|
9
9
|
generatedTokens?: number | undefined;
|
|
10
10
|
emittedTokens?: number | undefined;
|
|
11
11
|
avgConcurrentSeq?: number | undefined;
|
|
12
|
+
toolDefinitionsDropped?: number | undefined;
|
|
12
13
|
backendDevice?: "gpu" | "cpu" | undefined;
|
|
13
14
|
} | undefined;
|
|
14
15
|
/**
|
|
@@ -11,6 +11,7 @@ export function normalizeCompletionStats(stats) {
|
|
|
11
11
|
const promptTokens = finiteNumber(stats.promptTokens);
|
|
12
12
|
const generatedTokens = finiteNumber(stats.generatedTokens);
|
|
13
13
|
const avgConcurrentSeq = finiteNumber(stats.avgConcurrentSeq);
|
|
14
|
+
const toolDefinitionsDropped = finiteNumber(stats.toolDefinitionsDropped);
|
|
14
15
|
const normalized = {
|
|
15
16
|
...(timeToFirstToken !== undefined && { timeToFirstToken }),
|
|
16
17
|
...(tokensPerSecond !== undefined && { tokensPerSecond }),
|
|
@@ -19,6 +20,7 @@ export function normalizeCompletionStats(stats) {
|
|
|
19
20
|
...(promptTokens !== undefined && { promptTokens }),
|
|
20
21
|
...(generatedTokens !== undefined && { generatedTokens }),
|
|
21
22
|
...(avgConcurrentSeq !== undefined && { avgConcurrentSeq }),
|
|
23
|
+
...(toolDefinitionsDropped !== undefined && { toolDefinitionsDropped }),
|
|
22
24
|
...(stats.backendDevice !== undefined && { backendDevice: stats.backendDevice })
|
|
23
25
|
};
|
|
24
26
|
if (timeToFirstToken === undefined &&
|
|
@@ -28,6 +30,7 @@ export function normalizeCompletionStats(stats) {
|
|
|
28
30
|
promptTokens === undefined &&
|
|
29
31
|
generatedTokens === undefined &&
|
|
30
32
|
avgConcurrentSeq === undefined &&
|
|
33
|
+
toolDefinitionsDropped === undefined &&
|
|
31
34
|
stats.backendDevice === undefined) {
|
|
32
35
|
return undefined;
|
|
33
36
|
}
|
|
@@ -26,6 +26,15 @@ export declare function transformMessages(messages: Array<{
|
|
|
26
26
|
path: string;
|
|
27
27
|
}[] | undefined;
|
|
28
28
|
} | Tool>): ChatHistory[];
|
|
29
|
+
/**
|
|
30
|
+
* Put the model's configured system prompt in front of a history that carries
|
|
31
|
+
* none. `transform.ts` strips `system_prompt` from the config handed to the
|
|
32
|
+
* addon, so the conversation is the only way it reaches the model.
|
|
33
|
+
*/
|
|
34
|
+
export declare function seedConfiguredSystemPrompt<T extends {
|
|
35
|
+
role: string;
|
|
36
|
+
content: string;
|
|
37
|
+
}>(history: T[], modelConfig: unknown): T[];
|
|
29
38
|
export declare function completion(params: CompletionParams & {
|
|
30
39
|
tools?: Tool[];
|
|
31
40
|
generationParams?: GenerationParams;
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { logCacheDisabled,
|
|
1
|
+
import { logCacheDisabled, logCacheSave, logMessagesToAddon } from './cache-logger.js';
|
|
2
2
|
import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
|
|
3
|
-
import { isAddonPreMutationRefusal, isAddonContextOverflowError } from './context-overflow.js';
|
|
4
3
|
import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
|
|
5
4
|
import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
|
|
6
5
|
import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
|
|
7
6
|
import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
8
7
|
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
9
8
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
9
|
+
import { toolChoiceDemandsCall } from '../../../../schemas/completion-stream.js';
|
|
10
10
|
import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
|
|
11
11
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
12
12
|
import { AttachmentNotFoundError } from '../../../../errors/index.js';
|
|
@@ -57,90 +57,77 @@ export function transformMessages(messages) {
|
|
|
57
57
|
return transformed;
|
|
58
58
|
}
|
|
59
59
|
/**
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
* is not a renderable conversation for every chat template (Qwen3.5 raises
|
|
64
|
-
* `No user query found in messages.` because it anchors its tool block on the
|
|
65
|
-
* last user query), and a template failure degrades the whole render rather
|
|
66
|
-
* than just the tool block. Tools travel with the turn instead.
|
|
60
|
+
* Put the model's configured system prompt in front of a history that carries
|
|
61
|
+
* none. `transform.ts` strips `system_prompt` from the config handed to the
|
|
62
|
+
* addon, so the conversation is the only way it reaches the model.
|
|
67
63
|
*/
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
saveCacheToDisk: true,
|
|
75
|
-
prefill: true
|
|
76
|
-
});
|
|
77
|
-
// Register the prime so an abort during a cold-cache prefill cancels it.
|
|
78
|
-
onResponse?.(primeResponse);
|
|
79
|
-
await primeResponse.await();
|
|
64
|
+
export function seedConfiguredSystemPrompt(history, modelConfig) {
|
|
65
|
+
const configured = modelConfig.system_prompt;
|
|
66
|
+
if (!configured || extractSystemPrompt(history) !== null)
|
|
67
|
+
return history;
|
|
68
|
+
// Every caller's message type makes everything past role and content optional.
|
|
69
|
+
return [{ role: 'system', content: configured }, ...history];
|
|
80
70
|
}
|
|
81
71
|
/**
|
|
82
|
-
* Attach the tool block
|
|
83
|
-
*
|
|
72
|
+
* Attach the tool block to a turn payload, mirroring the no-kv-cache path
|
|
73
|
+
* (`prependToolsToHistory`): after a system message when the payload carries
|
|
74
|
+
* one, ahead of everything otherwise.
|
|
84
75
|
*/
|
|
85
76
|
function withToolBlock(messages, toolBlock) {
|
|
86
77
|
if (toolBlock.length === 0)
|
|
87
78
|
return messages;
|
|
88
|
-
|
|
79
|
+
const systemIndex = messages.findIndex((msg) => msg.role === 'system');
|
|
80
|
+
if (systemIndex === -1)
|
|
81
|
+
return [...toolBlock, ...messages];
|
|
82
|
+
return [...messages.slice(0, systemIndex + 1), ...toolBlock, ...messages.slice(systemIndex + 1)];
|
|
89
83
|
}
|
|
90
84
|
/**
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
85
|
+
* Fallback guess for whether a payload carrying this tool block gets it in
|
|
86
|
+
* front of the model, used only when the addon does not report
|
|
87
|
+
* `toolDefinitionsDropped`. Qwen-family templates anchor their tool section on
|
|
88
|
+
* the last user query and raise without one, and the addon answers that by
|
|
89
|
+
* re-rendering with tools stripped, so require a user message before believing
|
|
90
|
+
* the block landed.
|
|
97
91
|
*/
|
|
98
92
|
function rendersToolBlock(messages, toolBlock) {
|
|
99
93
|
if (toolBlock.length === 0)
|
|
100
94
|
return false;
|
|
101
95
|
return messages.some((msg) => msg.role === 'user');
|
|
102
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* Settle whether the committed prefix holds the full tool block. The addon's
|
|
99
|
+
* `toolDefinitionsDropped` is the template's own word on whether the block it
|
|
100
|
+
* was handed reached the model; without it (a payload with no block, or a
|
|
101
|
+
* stand-in model) the payload's estimate stands.
|
|
102
|
+
*/
|
|
103
|
+
function resolveToolBlockCached(payload, stats) {
|
|
104
|
+
const dropped = stats?.toolDefinitionsDropped;
|
|
105
|
+
if (!payload.toolBlockSent || dropped === undefined)
|
|
106
|
+
return payload.toolBlockCached;
|
|
107
|
+
return payload.prefixHoldsBlock || (payload.toolBlockFull && dropped === 0);
|
|
108
|
+
}
|
|
103
109
|
/**
|
|
104
110
|
* Pick the messages that need to reach the model for the next turn.
|
|
105
111
|
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
* turn
|
|
109
|
-
*
|
|
110
|
-
* - Empty history: nothing to slice; send whatever non-system messages
|
|
111
|
-
* exist. (The call site always reports the cache as existing, so this
|
|
112
|
-
* is the only way into this branch.)
|
|
113
|
-
* - Cache hit with a recorded `savedCount`: send only the unsaved tail
|
|
114
|
-
* (`history.slice(savedCount)`), so a multi-message turn (e.g. a
|
|
115
|
-
* consumer pushing both an assistant transcript and a follow-up user
|
|
116
|
-
* message between completions) all reaches the model.
|
|
117
|
-
* - Cache hit with a stale/missing `savedCount`: fall back to the full
|
|
118
|
-
* non-system history. The session is told (`dropStaleSavedCount`) so
|
|
119
|
-
* the bad boundary doesn't propagate into the next turn.
|
|
120
|
-
* - The tool block travels only with the turn that writes it into the
|
|
121
|
-
* cache; see `skipToolBlock` below.
|
|
112
|
+
* The cache holds whatever a committed turn sent, the tool block included, so
|
|
113
|
+
* the block travels with a turn rather than being written into the prefix on
|
|
114
|
+
* its own, and only with the turn that writes it into the cache — see
|
|
115
|
+
* `skipToolBlock` below.
|
|
122
116
|
*/
|
|
123
|
-
function prepareMessagesForCache(session, turn,
|
|
117
|
+
async function prepareMessagesForCache(session, turn, history, tools, toolChoice) {
|
|
124
118
|
const toolBlock = tools?.length ? transformMessages(tools) : [];
|
|
125
|
-
if (!(cacheExists && history.length > 0)) {
|
|
126
|
-
const historyWithoutSystem = history.filter((msg) => msg.role !== 'system');
|
|
127
|
-
return {
|
|
128
|
-
messages: withToolBlock(transformMessages(historyWithoutSystem), toolBlock),
|
|
129
|
-
toolBlockCached: rendersToolBlock(historyWithoutSystem, toolBlock)
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
119
|
// Slice from the turn's `savedCount` so callers can
|
|
133
120
|
// stage multiple messages between completions. `decideCachedHistorySlice`
|
|
134
121
|
// also guards against the QVAC-17780 stale-count regression: if the
|
|
135
122
|
// saved boundary would slice the history down to an empty payload
|
|
136
123
|
// (e.g. after a cancelled mid-decode), it falls back to the full
|
|
137
|
-
//
|
|
138
|
-
// The session owns the entry; `dropStaleSavedCount` clears it
|
|
139
|
-
// without touching the
|
|
140
|
-
// — only the boundary count is wrong).
|
|
141
|
-
const { messages, clearStaleCount } = decideCachedHistorySlice(turn.savedCount,
|
|
124
|
+
// history and signals the caller to drop the bad entry.
|
|
125
|
+
// The session owns the entry; `dropStaleSavedCount` clears it in memory
|
|
126
|
+
// and on disk without touching the cache file (the file is still
|
|
127
|
+
// trustworthy — only the boundary count is wrong).
|
|
128
|
+
const { messages, clearStaleCount } = decideCachedHistorySlice(turn.savedCount, history);
|
|
142
129
|
if (clearStaleCount) {
|
|
143
|
-
session.dropStaleSavedCount(turn);
|
|
130
|
+
await session.dropStaleSavedCount(turn);
|
|
144
131
|
}
|
|
145
132
|
// The block is never trimmed back out of the cache, so re-sending it every
|
|
146
133
|
// turn would leave one copy per turn and grow the prefix with the
|
|
@@ -148,14 +135,24 @@ function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
|
|
|
148
135
|
// one: `toolBlockCached` records that a previous turn actually got it into
|
|
149
136
|
// the cache, which a committed message count does not prove. A stale
|
|
150
137
|
// boundary means we are resending the whole conversation anyway.
|
|
151
|
-
const
|
|
138
|
+
const prefixHoldsBlock = turn.toolBlockCached && !clearStaleCount;
|
|
139
|
+
// The addon arms the tool-call grammar only for a payload that carries
|
|
140
|
+
// tools, and `required` / a named tool cannot be honoured without it. Such a
|
|
141
|
+
// turn resends the block even into a prefix that holds one; the second copy
|
|
142
|
+
// in the cache is the price of the guarantee.
|
|
143
|
+
const demandsCall = toolChoiceDemandsCall(toolChoice) && toolBlock.length > 0;
|
|
144
|
+
const skipToolBlock = prefixHoldsBlock && !demandsCall;
|
|
152
145
|
const blockToSend = skipToolBlock ? [] : toolBlock;
|
|
146
|
+
const toolBlockFull = blockToSend.length > 0 && (!demandsCall || toolChoice === 'required');
|
|
153
147
|
return {
|
|
154
148
|
messages: withToolBlock(transformMessages(messages), blockToSend),
|
|
155
|
-
|
|
149
|
+
toolBlockSent: blockToSend.length > 0,
|
|
150
|
+
toolBlockFull,
|
|
151
|
+
prefixHoldsBlock,
|
|
152
|
+
toolBlockCached: prefixHoldsBlock || (toolBlockFull && rendersToolBlock(messages, blockToSend))
|
|
156
153
|
};
|
|
157
154
|
}
|
|
158
|
-
async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse) {
|
|
155
|
+
async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse, onRunSettled) {
|
|
159
156
|
const runOptions = {
|
|
160
157
|
...(generationParams && { generationParams }),
|
|
161
158
|
...(cacheOptions?.cacheKey !== undefined && {
|
|
@@ -184,6 +181,8 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
|
|
|
184
181
|
yield { token: tokenStr };
|
|
185
182
|
}
|
|
186
183
|
const modelExecutionMs = nowMs() - modelStart;
|
|
184
|
+
// The addon has finished, and saved the cache file if it was going to.
|
|
185
|
+
onRunSettled?.();
|
|
187
186
|
if (cacheOptions?.saveCacheToDisk && cacheOptions.cacheKey) {
|
|
188
187
|
logCacheSave(cacheOptions.cacheKey);
|
|
189
188
|
}
|
|
@@ -193,19 +192,22 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
|
|
|
193
192
|
}
|
|
194
193
|
const responseWithStats = response;
|
|
195
194
|
const stats = withEmittedTokens(normalizeCompletionStats(responseWithStats.stats), emittedPieces);
|
|
195
|
+
const stopReason = responseWithStats.stats?.stopReason;
|
|
196
196
|
return {
|
|
197
197
|
...buildStreamResult(modelExecutionMs, stats),
|
|
198
198
|
toolCalls: toolCallsResult,
|
|
199
199
|
responseText: accumulatedText,
|
|
200
200
|
producedTokens,
|
|
201
|
-
stoppedAtContextBoundary:
|
|
201
|
+
stoppedAtContextBoundary: stopReason === 'contextOverflow',
|
|
202
|
+
generationFinished: stopReason !== 'none'
|
|
202
203
|
};
|
|
203
204
|
}
|
|
204
205
|
export async function* completion(params, opts) {
|
|
205
|
-
const { history, modelId, kvCache, tools, generationParams, responseFormat } = params;
|
|
206
|
+
const { history: requestHistory, modelId, kvCache, tools, generationParams, responseFormat } = params;
|
|
206
207
|
const { signal, scope } = opts;
|
|
207
208
|
const requestLogger = opts.logger ?? logger;
|
|
208
209
|
const modelConfig = getModelConfig(modelId);
|
|
210
|
+
const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
|
|
209
211
|
const toolsEnabled = modelConfig.tools === true;
|
|
210
212
|
const toolsActive = !!tools?.length && toolsEnabled;
|
|
211
213
|
const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
|
|
@@ -267,30 +269,25 @@ export async function* completion(params, opts) {
|
|
|
267
269
|
logMessagesToAddon(transformedHistory, 'NO_CACHE');
|
|
268
270
|
return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect, setActiveResponse);
|
|
269
271
|
}
|
|
270
|
-
// ---- KV-cache path. The session owns every bookkeeping layer; the
|
|
271
|
-
//
|
|
272
|
-
// `releaseTurn`
|
|
273
|
-
//
|
|
274
|
-
//
|
|
272
|
+
// ---- KV-cache path. The session owns every bookkeeping layer; the handler
|
|
273
|
+
// registers one deferred unwind that `commitTurn` short-circuits on the happy
|
|
274
|
+
// path. It is the non-destructive `releaseTurn` when the committed file is
|
|
275
|
+
// known to be intact — a throw before the addon run settled, or a cancel the
|
|
276
|
+
// addon rewound — and the destructive `rollback` for everything else,
|
|
277
|
+
// including zero-token replies, budget and context stops, and rename
|
|
278
|
+
// failures. ----
|
|
275
279
|
const session = createKvCacheSession(modelId, { logger: requestLogger });
|
|
276
280
|
const systemPromptFromHistory = extractSystemPrompt(history);
|
|
277
281
|
// The tool block is baked into the cache on the turn that first sends it and
|
|
278
282
|
// never trimmed, so a late or changed tool set has to land on a fresh cache
|
|
279
283
|
// rather than a warm prefix holding the old block.
|
|
280
284
|
const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? tools : undefined);
|
|
281
|
-
const systemPromptToUse = systemPromptFromHistory ||
|
|
282
|
-
modelConfig.system_prompt ||
|
|
283
|
-
'You are a helpful assistant.';
|
|
284
|
-
const primeIfMissing = async (cachePath) => {
|
|
285
|
-
await initSystemPromptCache(model, cachePath, systemPromptToUse, typeof kvCache === 'string' ? kvCache : 'auto', setActiveResponse);
|
|
286
|
-
};
|
|
287
285
|
let turn;
|
|
288
286
|
if (typeof kvCache === 'string') {
|
|
289
287
|
turn = await session.beginTurn({
|
|
290
288
|
kind: 'custom',
|
|
291
289
|
customKey: kvCache,
|
|
292
290
|
configHash,
|
|
293
|
-
primeIfMissing,
|
|
294
291
|
signal
|
|
295
292
|
});
|
|
296
293
|
}
|
|
@@ -304,7 +301,6 @@ export async function* completion(params, opts) {
|
|
|
304
301
|
kind: 'auto',
|
|
305
302
|
configHash,
|
|
306
303
|
history: cacheMessages,
|
|
307
|
-
primeIfMissing,
|
|
308
304
|
signal
|
|
309
305
|
});
|
|
310
306
|
}
|
|
@@ -312,17 +308,14 @@ export async function* completion(params, opts) {
|
|
|
312
308
|
// flips the turn's internal `committed` flag so this becomes a no-op
|
|
313
309
|
// on the happy path. Scope unwinding is LIFO — registered after the
|
|
314
310
|
// `removeEventListener` defer above so rollback runs before the
|
|
315
|
-
// listener detach.
|
|
316
|
-
//
|
|
311
|
+
// listener detach. `preserveCacheOnUnwind` selects the non-destructive
|
|
312
|
+
// `releaseTurn` when the file on disk is known to still hold the last
|
|
313
|
+
// committed turn; `releaseTurn` still drops a cache this turn created.
|
|
317
314
|
let preserveCacheOnUnwind = false;
|
|
318
315
|
scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
|
|
319
|
-
// `cacheExists` is implied by `beginTurn` — the session either found
|
|
320
|
-
// an existing cache or just primed one. Pass `true` to the message
|
|
321
|
-
// selector so the slicing branches engage.
|
|
322
316
|
let payload;
|
|
323
317
|
try {
|
|
324
|
-
payload = prepareMessagesForCache(session, turn,
|
|
325
|
-
/* cacheExists */ true, history, toolsActive ? tools : undefined);
|
|
318
|
+
payload = await prepareMessagesForCache(session, turn, history, toolsActive ? tools : undefined, mergedGenerationParams?.tool_choice);
|
|
326
319
|
}
|
|
327
320
|
catch (error) {
|
|
328
321
|
// A missing attachment is caller input rejected before the addon runs,
|
|
@@ -333,11 +326,18 @@ export async function* completion(params, opts) {
|
|
|
333
326
|
const messagesToSend = payload.messages;
|
|
334
327
|
logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
|
|
335
328
|
let result;
|
|
329
|
+
let addonRunSettled = false;
|
|
336
330
|
try {
|
|
337
|
-
result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse)
|
|
331
|
+
result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
|
|
332
|
+
addonRunSettled = true;
|
|
333
|
+
});
|
|
338
334
|
}
|
|
339
335
|
catch (error) {
|
|
340
|
-
|
|
336
|
+
// The addon writes the cache file only after a run completes and skips the
|
|
337
|
+
// save on every error path, so a run that threw left the committed file as
|
|
338
|
+
// it was. An engine-side throw after the run settled is the opposite: the
|
|
339
|
+
// file already holds this turn while no boundary was recorded for it.
|
|
340
|
+
preserveCacheOnUnwind = !addonRunSettled;
|
|
341
341
|
throw error;
|
|
342
342
|
}
|
|
343
343
|
const shouldCommitTurn = shouldCommitCachedTurn({
|
|
@@ -347,6 +347,13 @@ export async function* completion(params, opts) {
|
|
|
347
347
|
predict: mergedGenerationParams?.predict ?? modelConfig.predict,
|
|
348
348
|
stoppedAtContextBoundary: result.stoppedAtContextBoundary
|
|
349
349
|
});
|
|
350
|
+
// A cancelled run is rewound by the addon to the pre-request state before the
|
|
351
|
+
// file is re-saved, so the committed cache is intact. Every other non-commit
|
|
352
|
+
// finish (zero tokens, budget or context stop) was saved as-is and must go.
|
|
353
|
+
// An abort that landed after the addon reported a stop reason is the latter.
|
|
354
|
+
if (!shouldCommitTurn) {
|
|
355
|
+
preserveCacheOnUnwind = signal.aborted && !result.generationFinished;
|
|
356
|
+
}
|
|
350
357
|
if (typeof kvCache === 'string') {
|
|
351
358
|
// Custom-key path: the addon wrote the new cache state inline at
|
|
352
359
|
// the same path. Either commit (records the boundary, suppresses
|
|
@@ -355,7 +362,7 @@ export async function* completion(params, opts) {
|
|
|
355
362
|
await session.commitTurn(turn, {
|
|
356
363
|
kind: 'static',
|
|
357
364
|
messageCount: history.length + 1,
|
|
358
|
-
toolBlockCached: payload.
|
|
365
|
+
toolBlockCached: resolveToolBlockCached(payload, result.stats)
|
|
359
366
|
});
|
|
360
367
|
}
|
|
361
368
|
return result;
|
|
@@ -387,7 +394,7 @@ export async function* completion(params, opts) {
|
|
|
387
394
|
kind: 'autoRename',
|
|
388
395
|
targetCachePath: postResponseCacheInfo.cachePath,
|
|
389
396
|
messageCount: savedHistory.length,
|
|
390
|
-
toolBlockCached: payload.
|
|
397
|
+
toolBlockCached: resolveToolBlockCached(payload, result.stats)
|
|
391
398
|
});
|
|
392
399
|
return result;
|
|
393
400
|
}
|
|
@@ -15,7 +15,6 @@
|
|
|
15
15
|
* + extraction logic without a real model load.
|
|
16
16
|
*/
|
|
17
17
|
export declare function isAddonContextOverflowError(err: unknown): boolean;
|
|
18
|
-
export declare function isAddonPreMutationRefusal(err: unknown): boolean;
|
|
19
18
|
export type ContextOverflowSizes = {
|
|
20
19
|
/** The prompt alone, in tokens; unset when the guard reports KV cells. */
|
|
21
20
|
promptTokens?: number;
|
|
@@ -48,37 +48,6 @@ export function isAddonContextOverflowError(err) {
|
|
|
48
48
|
const trimmed = message.trim();
|
|
49
49
|
return CONTEXT_OVERFLOW_FORMS.some((form) => form.test(trimmed));
|
|
50
50
|
}
|
|
51
|
-
// Refusals thrown before any decode or disk save; the async transport delivers
|
|
52
|
-
// exception.what() alone, so each form is matched complete and end-anchored.
|
|
53
|
-
const PRE_MUTATION_REFUSAL_FORMS = [
|
|
54
|
-
/^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
55
|
-
/^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
56
|
-
/^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
57
|
-
/^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
58
|
-
/^ContinuousBatchScheduler::submit: failed to add to batch \(MultiRequestBatcher::AddStatus=[1-4]\)$/,
|
|
59
|
-
/^invalid generationParams\.json_schema: [^\r\n]*$/,
|
|
60
|
-
/^failed to initialise sampler with per-request generationParams \(invalid grammar or json_schema\?\)$/,
|
|
61
|
-
/^\[MtmdLlm\] Media buffer is empty$/,
|
|
62
|
-
/^\[MtmdLlm\] Filename is empty$/,
|
|
63
|
-
/^\[MtmdLlm\] Failed to load media from file: [^\r\n]*$/,
|
|
64
|
-
/^\[MtmdLlm\] preparePrefill: prompt must end with text after the last media item$/
|
|
65
|
-
];
|
|
66
|
-
export function isAddonPreMutationRefusal(err) {
|
|
67
|
-
if (typeof err !== 'object' || err === null)
|
|
68
|
-
return false;
|
|
69
|
-
// A status code only exists on the synchronous throw path; when present it
|
|
70
|
-
// must be InvalidArgument, and its absence (the async transport) is fine.
|
|
71
|
-
const code = err.code;
|
|
72
|
-
if (code !== undefined &&
|
|
73
|
-
(typeof code !== 'string' || !/^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code))) {
|
|
74
|
-
return false;
|
|
75
|
-
}
|
|
76
|
-
const message = err.message;
|
|
77
|
-
if (typeof message !== 'string')
|
|
78
|
-
return false;
|
|
79
|
-
const trimmed = message.trim();
|
|
80
|
-
return PRE_MUTATION_REFUSAL_FORMS.some((form) => form.test(trimmed));
|
|
81
|
-
}
|
|
82
51
|
// One entry per number-formatting guard, most specific first; keep in step
|
|
83
52
|
// with TextLlmContext.cpp / MtmdLlmContext.cpp or fields silently vanish.
|
|
84
53
|
const MESSAGE_PATTERNS = [
|