@qvac/inference 0.21.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +30 -1
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +4 -1
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +5 -4
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +80 -28
- package/dist/handlers/get-loaded-model-info.js +1 -2
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +3 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/logger.js +1 -1
- package/dist/logging/utils.js +1 -1
- package/dist/models/registry/models.d.ts +563 -816
- package/dist/models/registry/models.js +2873 -3110
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +203 -2
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
- package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
- package/dist/plugins/builtin/index.d.ts +2 -0
- package/dist/plugins/builtin/index.js +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
- package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
- package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +10 -25
- package/dist/plugins/ops/load-model.js +21 -16
- package/dist/plugins/ops/transcribe.js +17 -1
- package/dist/registry.js +1 -0
- package/dist/resources/model-fit/assess.d.ts +15 -23
- package/dist/resources/model-fit/assess.js +207 -271
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
- package/dist/resources/model-fit/floor.d.ts +10 -10
- package/dist/resources/model-fit/floor.js +13 -13
- package/dist/resources/model-fit/kv-cache.d.ts +23 -0
- package/dist/resources/model-fit/kv-cache.js +85 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
- package/dist/resources/model-fit/platform.d.ts +4 -4
- package/dist/resources/model-fit/platform.js +6 -8
- package/dist/resources/model-fit/types.d.ts +3 -117
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/model-config-utils.js +6 -1
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/request-context.js +8 -0
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +100 -43
- package/dist/schemas/assess-model-fit.js +66 -29
- package/dist/schemas/audio-gen.d.ts +186 -18
- package/dist/schemas/audio-gen.js +180 -48
- package/dist/schemas/batch-completion-stream.d.ts +202 -34
- package/dist/schemas/bci-config.d.ts +2 -0
- package/dist/schemas/bci.d.ts +23 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +1083 -158
- package/dist/schemas/common.js +5 -2
- package/dist/schemas/completion-stream.d.ts +232 -37
- package/dist/schemas/completion-stream.js +64 -20
- package/dist/schemas/config.d.ts +477 -12
- package/dist/schemas/config.js +21 -1
- package/dist/schemas/decision.d.ts +602 -0
- package/dist/schemas/decision.js +205 -0
- package/dist/schemas/download-asset.d.ts +7 -3
- package/dist/schemas/engine-addon-map.d.ts +7 -0
- package/dist/schemas/engine-addon-map.js +23 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.js +2 -0
- package/dist/schemas/llamacpp-config.d.ts +42 -2
- package/dist/schemas/llamacpp-config.js +27 -0
- package/dist/schemas/load-model.d.ts +5247 -1309
- package/dist/schemas/load-model.js +88 -9
- package/dist/schemas/model-config-schemas.d.ts +233 -8
- package/dist/schemas/model-config-schemas.js +4 -1
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +2 -1
- package/dist/schemas/model-resource-profile.d.ts +3 -4
- package/dist/schemas/model-resource-profile.js +1 -4
- package/dist/schemas/model-src-utils.d.ts +10 -2
- package/dist/schemas/model-types.d.ts +10 -1
- package/dist/schemas/model-types.js +3 -0
- package/dist/schemas/ocr.d.ts +74 -0
- package/dist/schemas/ocr.js +42 -0
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/registry.d.ts +26 -0
- package/dist/schemas/registry.js +3 -0
- package/dist/schemas/reload-config.d.ts +10 -2
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +26 -0
- package/dist/schemas/sdcpp-config.js +8 -7
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +402 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +15 -0
- package/dist/schemas/transcription-config.js +38 -3
- package/dist/schemas/transcription.d.ts +46 -4
- package/dist/schemas/transcription.js +51 -4
- package/dist/schemas/translation-config.d.ts +30 -10
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/mcp-adapter.js +12 -8
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +1 -0
- package/package.json +41 -32
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { logCacheDisabled, logCacheSave, logMessagesToAddon } from './cache-logger.js';
|
|
2
2
|
import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
|
|
3
3
|
import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
|
|
4
|
-
import {
|
|
4
|
+
import { shouldCommitCachedTurn } from './kv-cache-state.js';
|
|
5
5
|
import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
|
|
6
6
|
import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
7
7
|
import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
|
|
8
8
|
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
9
9
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
10
|
-
import { toolChoiceDemandsCall } from '../../../../schemas/completion-stream.js';
|
|
11
10
|
import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
|
|
12
11
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
13
12
|
import { AttachmentNotFoundError } from '../../../../errors/index.js';
|
|
@@ -43,7 +42,16 @@ function transformMessage(message) {
|
|
|
43
42
|
}
|
|
44
43
|
transformed.push({
|
|
45
44
|
role: msg.role,
|
|
46
|
-
content: msg.content
|
|
45
|
+
content: msg.content,
|
|
46
|
+
...(msg.toolCalls && {
|
|
47
|
+
tool_calls: msg.toolCalls.map((call) => ({
|
|
48
|
+
...(call.id !== undefined && { id: call.id }),
|
|
49
|
+
name: call.name,
|
|
50
|
+
arguments: call.arguments
|
|
51
|
+
}))
|
|
52
|
+
}),
|
|
53
|
+
...(msg.toolCallId !== undefined && { tool_call_id: msg.toolCallId }),
|
|
54
|
+
...(msg.toolName !== undefined && { name: msg.toolName })
|
|
47
55
|
});
|
|
48
56
|
return transformed;
|
|
49
57
|
}
|
|
@@ -57,6 +65,18 @@ export function transformMessages(messages) {
|
|
|
57
65
|
}
|
|
58
66
|
return transformed;
|
|
59
67
|
}
|
|
68
|
+
// The fields that identify a turn, so two tool turns with equal `content`
|
|
69
|
+
// never share a cache key.
|
|
70
|
+
function toCacheMessage(msg) {
|
|
71
|
+
return {
|
|
72
|
+
role: msg.role,
|
|
73
|
+
content: msg.content,
|
|
74
|
+
attachments: msg.attachments ?? undefined,
|
|
75
|
+
...(msg.toolCalls && { toolCalls: msg.toolCalls }),
|
|
76
|
+
...(msg.toolCallId !== undefined && { toolCallId: msg.toolCallId }),
|
|
77
|
+
...(msg.toolName !== undefined && { toolName: msg.toolName })
|
|
78
|
+
};
|
|
79
|
+
}
|
|
60
80
|
/**
|
|
61
81
|
* Put the model's configured system prompt in front of a history that carries
|
|
62
82
|
* none. `transform.ts` strips `system_prompt` from the config handed to the
|
|
@@ -69,98 +89,51 @@ export function seedConfiguredSystemPrompt(history, modelConfig) {
|
|
|
69
89
|
// Every caller's message type makes everything past role and content optional.
|
|
70
90
|
return [{ role: 'system', content: configured }, ...history];
|
|
71
91
|
}
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
function withToolBlock(messages, toolBlock) {
|
|
78
|
-
if (toolBlock.length === 0)
|
|
79
|
-
return messages;
|
|
80
|
-
const systemIndex = messages.findIndex((msg) => msg.role === 'system');
|
|
81
|
-
if (systemIndex === -1)
|
|
82
|
-
return [...toolBlock, ...messages];
|
|
83
|
-
return [...messages.slice(0, systemIndex + 1), ...toolBlock, ...messages.slice(systemIndex + 1)];
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Fallback guess for whether a payload carrying this tool block gets it in
|
|
87
|
-
* front of the model, used only when the addon does not report
|
|
88
|
-
* `toolDefinitionsDropped`. Qwen-family templates anchor their tool section on
|
|
89
|
-
* the last user query and raise without one, and the addon answers that by
|
|
90
|
-
* re-rendering with tools stripped, so require a user message before believing
|
|
91
|
-
* the block landed.
|
|
92
|
-
*/
|
|
93
|
-
function rendersToolBlock(messages, toolBlock) {
|
|
94
|
-
if (toolBlock.length === 0)
|
|
92
|
+
// The addon refuses to save a key it holds nothing for — a cold turn it rolled
|
|
93
|
+
// back — with `InvalidArgument`. As with its other status errors, an
|
|
94
|
+
// asynchronous rejection carries the message alone.
|
|
95
|
+
function isNothingCachedRejection(error) {
|
|
96
|
+
if (typeof error !== 'object' || error === null)
|
|
95
97
|
return false;
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
* stand-in model) the payload's estimate stands.
|
|
103
|
-
*/
|
|
104
|
-
function resolveToolBlockCached(payload, stats) {
|
|
105
|
-
const dropped = stats?.toolDefinitionsDropped;
|
|
106
|
-
if (!payload.toolBlockSent || dropped === undefined)
|
|
107
|
-
return payload.toolBlockCached;
|
|
108
|
-
return payload.prefixHoldsBlock || (payload.toolBlockFull && dropped === 0);
|
|
98
|
+
const code = error.code;
|
|
99
|
+
if (code !== undefined) {
|
|
100
|
+
return typeof code === 'string' && /^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code);
|
|
101
|
+
}
|
|
102
|
+
const message = error.message;
|
|
103
|
+
return typeof message === 'string' && /no conversation is cached/.test(message);
|
|
109
104
|
}
|
|
110
105
|
/**
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
106
|
+
* Write the conversation the addon holds for `cachePath` to that file. The
|
|
107
|
+
* session commits on the file, and a saved conversation is one the addon drops
|
|
108
|
+
* once `deleteCache` removes the file. A cold turn the addon rolled back has
|
|
109
|
+
* nothing to write, and the commit then finds no file. Any other rejection is
|
|
110
|
+
* a write that failed: the file is behind the conversation, which the next
|
|
111
|
+
* turn re-decodes.
|
|
117
112
|
*/
|
|
118
|
-
async function
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
113
|
+
async function saveCacheFile(model, cachePath, log) {
|
|
114
|
+
const saveCache = model.saveCache;
|
|
115
|
+
if (!saveCache) {
|
|
116
|
+
log.debug(`[kv-cache] model has no saveCache; nothing written for ${cachePath}`);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
try {
|
|
120
|
+
await saveCache.call(model, cachePath);
|
|
121
|
+
logCacheSave(cachePath);
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
125
|
+
if (isNothingCachedRejection(error)) {
|
|
126
|
+
log.debug(`[kv-cache] saveCache had nothing to write for ${cachePath}: ${detail}`);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
log.warn(`[kv-cache] saveCache failed; the file is behind the conversation and the next turn re-decodes what it lacks. path=${cachePath} error=${detail}`);
|
|
132
130
|
}
|
|
133
|
-
// The block is never trimmed back out of the cache, so re-sending it every
|
|
134
|
-
// turn would leave one copy per turn and grow the prefix with the
|
|
135
|
-
// conversation. Skip it only when the prefix is known to hold a rendered
|
|
136
|
-
// one: `toolBlockCached` records that a previous turn actually got it into
|
|
137
|
-
// the cache, which a committed message count does not prove. A stale
|
|
138
|
-
// boundary means we are resending the whole conversation anyway.
|
|
139
|
-
const prefixHoldsBlock = turn.toolBlockCached && !clearStaleCount;
|
|
140
|
-
// The addon arms the tool-call grammar only for a payload that carries
|
|
141
|
-
// tools, and `required` / a named tool cannot be honoured without it. Such a
|
|
142
|
-
// turn resends the block even into a prefix that holds one; the second copy
|
|
143
|
-
// in the cache is the price of the guarantee.
|
|
144
|
-
const demandsCall = toolChoiceDemandsCall(toolChoice) && toolBlock.length > 0;
|
|
145
|
-
const skipToolBlock = prefixHoldsBlock && !demandsCall;
|
|
146
|
-
const blockToSend = skipToolBlock ? [] : toolBlock;
|
|
147
|
-
const toolBlockFull = blockToSend.length > 0 && (!demandsCall || toolChoice === 'required');
|
|
148
|
-
return {
|
|
149
|
-
messages: withToolBlock(transformMessages(messages), blockToSend),
|
|
150
|
-
toolBlockSent: blockToSend.length > 0,
|
|
151
|
-
toolBlockFull,
|
|
152
|
-
prefixHoldsBlock,
|
|
153
|
-
toolBlockCached: prefixHoldsBlock || (toolBlockFull && rendersToolBlock(messages, blockToSend))
|
|
154
|
-
};
|
|
155
131
|
}
|
|
156
|
-
async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse
|
|
132
|
+
async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse) {
|
|
157
133
|
const runOptions = {
|
|
158
134
|
...(generationParams && { generationParams }),
|
|
159
135
|
...(cacheOptions?.cacheKey !== undefined && {
|
|
160
136
|
cacheKey: cacheOptions.cacheKey
|
|
161
|
-
}),
|
|
162
|
-
...(cacheOptions?.saveCacheToDisk !== undefined && {
|
|
163
|
-
saveCacheToDisk: cacheOptions.saveCacheToDisk
|
|
164
137
|
})
|
|
165
138
|
};
|
|
166
139
|
const hasRunOptions = Object.keys(runOptions).length > 0;
|
|
@@ -182,11 +155,6 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
|
|
|
182
155
|
yield { token: tokenStr };
|
|
183
156
|
}
|
|
184
157
|
const modelExecutionMs = nowMs() - modelStart;
|
|
185
|
-
// The addon has finished, and saved the cache file if it was going to.
|
|
186
|
-
onRunSettled?.();
|
|
187
|
-
if (cacheOptions?.saveCacheToDisk && cacheOptions.cacheKey) {
|
|
188
|
-
logCacheSave(cacheOptions.cacheKey);
|
|
189
|
-
}
|
|
190
158
|
if (tools && tools.length > 0) {
|
|
191
159
|
const { toolCalls } = parseToolCalls(accumulatedText, tools, dialect);
|
|
192
160
|
toolCallsResult = toolCalls;
|
|
@@ -199,8 +167,7 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
|
|
|
199
167
|
toolCalls: toolCallsResult,
|
|
200
168
|
responseText: accumulatedText,
|
|
201
169
|
producedTokens,
|
|
202
|
-
stoppedAtContextBoundary: stopReason === 'contextOverflow'
|
|
203
|
-
generationFinished: stopReason !== 'none'
|
|
170
|
+
stoppedAtContextBoundary: stopReason === 'contextOverflow'
|
|
204
171
|
};
|
|
205
172
|
}
|
|
206
173
|
export async function* completion(params, opts) {
|
|
@@ -267,30 +234,30 @@ export async function* completion(params, opts) {
|
|
|
267
234
|
// Drop the ref so a late abort can't cancel an already-settled response.
|
|
268
235
|
activeResponse = null;
|
|
269
236
|
});
|
|
237
|
+
// The cached and uncached paths send the same prompt: the whole conversation
|
|
238
|
+
// and the tools, every turn. With a cache key the addon keeps the longest
|
|
239
|
+
// prefix it already holds and decodes only the rest.
|
|
240
|
+
const prompt = transformMessages(toolsActive && toolsToRender ? prependToolsToHistory(history, toolsToRender) : history);
|
|
270
241
|
if (!kvCache) {
|
|
271
|
-
// KV-cache disabled — straight passthrough, no session involvement.
|
|
272
|
-
let historyWithTools = history;
|
|
273
|
-
if (toolsActive && toolsToRender) {
|
|
274
|
-
historyWithTools = prependToolsToHistory(history, toolsToRender);
|
|
275
|
-
}
|
|
276
|
-
const transformedHistory = transformMessages(historyWithTools);
|
|
277
242
|
logCacheDisabled();
|
|
278
|
-
logMessagesToAddon(
|
|
279
|
-
return yield* processModelResponse(model,
|
|
243
|
+
logMessagesToAddon(prompt, 'NO_CACHE');
|
|
244
|
+
return yield* processModelResponse(model, prompt, callableTools, mergedGenerationParams, undefined, dialect, setActiveResponse);
|
|
280
245
|
}
|
|
281
246
|
// ---- KV-cache path. The session owns every bookkeeping layer; the handler
|
|
282
|
-
// registers one deferred unwind that `commitTurn` short-circuits
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
//
|
|
287
|
-
//
|
|
288
|
-
const
|
|
289
|
-
const
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
247
|
+
// registers one deferred unwind that `commitTurn` short-circuits. The addon
|
|
248
|
+
// leaves the file consistent whatever the outcome — it commits or rolls back
|
|
249
|
+
// the request itself — so the unwind keeps it (`releaseTurn`) except where
|
|
250
|
+
// the file has no key to live under. A rollback also drops the addon's copy
|
|
251
|
+
// of the conversation, which a later key switch or unload would otherwise
|
|
252
|
+
// write back to the deleted path. ----
|
|
253
|
+
const discardCache = model.discardCache;
|
|
254
|
+
const session = createKvCacheSession(modelId, {
|
|
255
|
+
logger: requestLogger,
|
|
256
|
+
...(discardCache && {
|
|
257
|
+
discardCache: (cachePath) => discardCache.call(model, cachePath)
|
|
258
|
+
})
|
|
259
|
+
});
|
|
260
|
+
const configHash = generateConfigHash(extractSystemPrompt(history));
|
|
294
261
|
let turn;
|
|
295
262
|
if (typeof kvCache === 'string') {
|
|
296
263
|
turn = await session.beginTurn({
|
|
@@ -301,11 +268,7 @@ export async function* completion(params, opts) {
|
|
|
301
268
|
});
|
|
302
269
|
}
|
|
303
270
|
else {
|
|
304
|
-
const cacheMessages = history.map(
|
|
305
|
-
role: msg.role,
|
|
306
|
-
content: msg.content,
|
|
307
|
-
attachments: msg.attachments ?? undefined
|
|
308
|
-
}));
|
|
271
|
+
const cacheMessages = history.map(toCacheMessage);
|
|
309
272
|
turn = await session.beginTurn({
|
|
310
273
|
kind: 'auto',
|
|
311
274
|
configHash,
|
|
@@ -313,67 +276,17 @@ export async function* completion(params, opts) {
|
|
|
313
276
|
signal
|
|
314
277
|
});
|
|
315
278
|
}
|
|
316
|
-
//
|
|
317
|
-
//
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
// committed turn; `releaseTurn` still drops a cache this turn created.
|
|
323
|
-
let preserveCacheOnUnwind = false;
|
|
324
|
-
scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
|
|
325
|
-
let payload;
|
|
326
|
-
try {
|
|
327
|
-
payload = await prepareMessagesForCache(session, turn, history, toolsActive ? toolsToRender : undefined, mergedGenerationParams?.tool_choice);
|
|
328
|
-
}
|
|
329
|
-
catch (error) {
|
|
330
|
-
// A missing attachment is caller input rejected before the addon runs,
|
|
331
|
-
// so the committed cache is untouched and must survive.
|
|
332
|
-
preserveCacheOnUnwind = error instanceof AttachmentNotFoundError;
|
|
333
|
-
throw error;
|
|
334
|
-
}
|
|
335
|
-
const messagesToSend = payload.messages;
|
|
336
|
-
logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
|
|
337
|
-
let result;
|
|
338
|
-
let addonRunSettled = false;
|
|
339
|
-
try {
|
|
340
|
-
result = yield* processModelResponse(model, messagesToSend, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
|
|
341
|
-
addonRunSettled = true;
|
|
342
|
-
});
|
|
343
|
-
}
|
|
344
|
-
catch (error) {
|
|
345
|
-
// The addon writes the cache file only after a run completes and skips the
|
|
346
|
-
// save on every error path, so a run that threw left the committed file as
|
|
347
|
-
// it was. An engine-side throw after the run settled is the opposite: the
|
|
348
|
-
// file already holds this turn while no boundary was recorded for it.
|
|
349
|
-
preserveCacheOnUnwind = !addonRunSettled;
|
|
350
|
-
throw error;
|
|
351
|
-
}
|
|
352
|
-
const shouldCommitTurn = shouldCommitCachedTurn({
|
|
353
|
-
aborted: signal.aborted,
|
|
354
|
-
producedTokens: result.producedTokens,
|
|
355
|
-
generatedTokens: result.stats?.generatedTokens,
|
|
356
|
-
predict: mergedGenerationParams?.predict ?? modelConfig.predict,
|
|
357
|
-
stoppedAtContextBoundary: result.stoppedAtContextBoundary
|
|
358
|
-
});
|
|
359
|
-
// A cancelled run is rewound by the addon to the pre-request state before the
|
|
360
|
-
// file is re-saved, so the committed cache is intact. Every other non-commit
|
|
361
|
-
// finish (zero tokens, budget or context stop) was saved as-is and must go.
|
|
362
|
-
// An abort that landed after the addon reported a stop reason is the latter.
|
|
363
|
-
if (!shouldCommitTurn) {
|
|
364
|
-
preserveCacheOnUnwind = signal.aborted && !result.generationFinished;
|
|
365
|
-
}
|
|
279
|
+
// Scope unwinding is LIFO — registered after the `removeEventListener`
|
|
280
|
+
// defer above so this runs before the listener detach.
|
|
281
|
+
let rollbackOnUnwind = false;
|
|
282
|
+
scope.defer(() => (rollbackOnUnwind ? session.rollback(turn) : session.releaseTurn(turn)));
|
|
283
|
+
logMessagesToAddon(prompt, 'PROMPT_SEND');
|
|
284
|
+
const result = yield* processModelResponse(model, prompt, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath }, dialect, setActiveResponse);
|
|
366
285
|
if (typeof kvCache === 'string') {
|
|
367
|
-
// Custom-key path: the
|
|
368
|
-
//
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
await session.commitTurn(turn, {
|
|
372
|
-
kind: 'static',
|
|
373
|
-
messageCount: history.length + 1,
|
|
374
|
-
toolBlockCached: resolveToolBlockCached(payload, result.stats)
|
|
375
|
-
});
|
|
376
|
-
}
|
|
286
|
+
// Custom-key path: the file holds whatever the addon kept, cancelled and
|
|
287
|
+
// stopped turns included.
|
|
288
|
+
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
289
|
+
await session.commitTurn(turn, { kind: 'static' });
|
|
377
290
|
return result;
|
|
378
291
|
}
|
|
379
292
|
// Auto-cache path.
|
|
@@ -381,29 +294,36 @@ export async function* completion(params, opts) {
|
|
|
381
294
|
// Tool-call turns: the auto-cache key is derived from
|
|
382
295
|
// `result.responseText`, which here is raw tool-call markup rather
|
|
383
296
|
// than a clean assistant message. There's no safe post-response key
|
|
384
|
-
// to rename to, so
|
|
297
|
+
// to rename to, so the deferred rollback drops the turn, unwritten. Once
|
|
385
298
|
// we support auto-cache for structured assistant/tool turns,
|
|
386
299
|
// this becomes a normal commit path.
|
|
387
300
|
if (result.toolCalls.length > 0) {
|
|
388
301
|
logger.warn(`[kv-cache] Auto cache tool-call turn; rolling back to avoid disk leak. path=${turn.cachePath}`);
|
|
302
|
+
rollbackOnUnwind = true;
|
|
389
303
|
return result;
|
|
390
304
|
}
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
305
|
+
const shouldRename = shouldCommitCachedTurn({
|
|
306
|
+
aborted: signal.aborted,
|
|
307
|
+
producedTokens: result.producedTokens,
|
|
308
|
+
generatedTokens: result.stats?.generatedTokens,
|
|
309
|
+
predict: mergedGenerationParams?.predict ?? modelConfig.predict,
|
|
310
|
+
stoppedAtContextBoundary: result.stoppedAtContextBoundary
|
|
311
|
+
});
|
|
312
|
+
if (!shouldRename) {
|
|
313
|
+
// A cancelled, empty or cut-off reply is not one the caller will send
|
|
314
|
+
// back as-is, so there is no post-response key to move to. The file stays
|
|
315
|
+
// under the history it was found by, holding what the addon kept. A first
|
|
316
|
+
// turn found no file, and one it wrote now would only be rolled back.
|
|
317
|
+
if (!turn.createdCache)
|
|
318
|
+
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
394
319
|
return result;
|
|
395
320
|
}
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
content: msg.content,
|
|
399
|
-
attachments: msg.attachments ?? undefined
|
|
400
|
-
})), result.responseText);
|
|
321
|
+
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
322
|
+
const savedHistory = buildAutoCacheSaveHistory(history.map(toCacheMessage), result.responseText);
|
|
401
323
|
const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory);
|
|
402
324
|
await session.commitTurn(turn, {
|
|
403
325
|
kind: 'autoRename',
|
|
404
|
-
targetCachePath: postResponseCacheInfo.cachePath
|
|
405
|
-
messageCount: savedHistory.length,
|
|
406
|
-
toolBlockCached: resolveToolBlockCached(payload, result.stats)
|
|
326
|
+
targetCachePath: postResponseCacheInfo.cachePath
|
|
407
327
|
});
|
|
408
328
|
return result;
|
|
409
329
|
}
|
|
@@ -2,51 +2,20 @@ import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
|
|
|
2
2
|
import { type CacheMessage } from '../../../../utils/index.js';
|
|
3
3
|
import type { Logger } from '../../../../logging/types.js';
|
|
4
4
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
5
|
-
import { z } from 'zod';
|
|
6
|
-
/**
|
|
7
|
-
* Coordinates six KV-cache state layers:
|
|
8
|
-
*
|
|
9
|
-
* 1. `cachedPrefixes` — saved message boundaries, and whether a tool
|
|
10
|
-
* block was rendered into them.
|
|
11
|
-
* 2. `initializedCaches` — caches this process has established.
|
|
12
|
-
* 3. On-disk `.bin` files written by the addon.
|
|
13
|
-
* 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
|
|
14
|
-
* boundary outlives the process; see `readPrefixSidecar`.
|
|
15
|
-
* 5. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
16
|
-
* 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
17
|
-
*
|
|
18
|
-
* Every turn must finish through `commitTurn`, `rollback`, or the
|
|
19
|
-
* non-destructive `releaseTurn` so all inference state stays aligned,
|
|
20
|
-
* the active-path ref is released, and marker metadata follows the
|
|
21
|
-
* cache directory lifecycle.
|
|
22
|
-
*/
|
|
23
|
-
/** What the kv-cache file at a given path is known to hold. */
|
|
24
|
-
declare const cachedPrefixSchema: z.ZodObject<{
|
|
25
|
-
messages: z.ZodNumber;
|
|
26
|
-
toolBlock: z.ZodBoolean;
|
|
27
|
-
}, z.core.$strip>;
|
|
28
|
-
type CachedPrefix = z.infer<typeof cachedPrefixSchema>;
|
|
29
5
|
export interface TurnHandle {
|
|
30
6
|
/** Resolved on-disk cache file path the addon will read from / write to. */
|
|
31
7
|
readonly cachePath: string;
|
|
32
8
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* message tail for the next addon call.
|
|
36
|
-
*/
|
|
37
|
-
readonly savedCount: number;
|
|
38
|
-
/**
|
|
39
|
-
* Whether the cached prefix already holds a rendered static tool block, so
|
|
40
|
-
* this turn can leave it out of its payload. False on a cold turn and
|
|
41
|
-
* whenever the previous turn couldn't confirm the block reached the model.
|
|
9
|
+
* True when no cache existed at this path when the turn began: nothing
|
|
10
|
+
* committed is there to keep, so an unwind drops whatever this turn wrote.
|
|
42
11
|
*/
|
|
43
|
-
readonly
|
|
12
|
+
readonly createdCache: boolean;
|
|
44
13
|
}
|
|
45
14
|
export interface BeginCustomTurnInput {
|
|
46
15
|
kind: 'custom';
|
|
47
16
|
/** User-provided session key (`completion({ kvCache: "session-a" })`). */
|
|
48
17
|
customKey: string;
|
|
49
|
-
/** Hash of system prompt
|
|
18
|
+
/** Hash of the system prompt; see `generateConfigHash`. */
|
|
50
19
|
configHash: string;
|
|
51
20
|
/**
|
|
52
21
|
* Request abort signal. When it aborts while this turn is queued behind a
|
|
@@ -57,7 +26,7 @@ export interface BeginCustomTurnInput {
|
|
|
57
26
|
}
|
|
58
27
|
export interface BeginAutoTurnInput {
|
|
59
28
|
kind: 'auto';
|
|
60
|
-
/** Hash of system prompt
|
|
29
|
+
/** Hash of the system prompt; see `generateConfigHash`. */
|
|
61
30
|
configHash: string;
|
|
62
31
|
/** Conversation history used to compute the pre-response cache key. */
|
|
63
32
|
history: CacheMessage[];
|
|
@@ -65,30 +34,16 @@ export interface BeginAutoTurnInput {
|
|
|
65
34
|
signal?: AbortSignal;
|
|
66
35
|
}
|
|
67
36
|
export type BeginTurnInput = BeginCustomTurnInput | BeginAutoTurnInput;
|
|
68
|
-
|
|
69
|
-
* Whether the committed prefix holds a rendered static tool block. The
|
|
70
|
-
* handler owns this because only it knows whether the payload carried the
|
|
71
|
-
* block and whether the message list was one the template renders tools for.
|
|
72
|
-
*/
|
|
73
|
-
interface ToolBlockCommit {
|
|
74
|
-
toolBlockCached: boolean;
|
|
75
|
-
}
|
|
76
|
-
export interface StaticCommitResult extends ToolBlockCommit {
|
|
37
|
+
export interface StaticCommitResult {
|
|
77
38
|
kind: 'static';
|
|
78
|
-
/** `history.length + 1` — recorded at the turn's current `cachePath`. */
|
|
79
|
-
messageCount: number;
|
|
80
39
|
}
|
|
81
|
-
export interface AutoRenameCommitResult
|
|
40
|
+
export interface AutoRenameCommitResult {
|
|
82
41
|
kind: 'autoRename';
|
|
83
42
|
/**
|
|
84
43
|
* Destination path the addon's pre-response cache file should be
|
|
85
|
-
* renamed to (computed from `cacheMessages + responseText`).
|
|
86
|
-
* stale entry at the source path is dropped from `cachedPrefixes`
|
|
87
|
-
* and the new count is recorded at this target path.
|
|
44
|
+
* renamed to (computed from `cacheMessages + responseText`).
|
|
88
45
|
*/
|
|
89
46
|
targetCachePath: string;
|
|
90
|
-
/** Number of messages the renamed cache represents (`savedHistory.length`). */
|
|
91
|
-
messageCount: number;
|
|
92
47
|
}
|
|
93
48
|
export type CommitResult = StaticCommitResult | AutoRenameCommitResult;
|
|
94
49
|
export interface KvCacheSession {
|
|
@@ -96,53 +51,48 @@ export interface KvCacheSession {
|
|
|
96
51
|
* Open a new turn against the cache. Resolves the cache file path,
|
|
97
52
|
* ensures its parent directory exists, and returns a `TurnHandle` the
|
|
98
53
|
* handler attaches to `ctx.scope.defer(...)` for the rollback hook. The
|
|
99
|
-
* addon's
|
|
100
|
-
*
|
|
54
|
+
* addon's `saveCache` writes the file after the run, so a cold turn opens
|
|
55
|
+
* without one. Auto-cache path resolution is serialized with
|
|
101
56
|
* retention deletion before the handle is returned.
|
|
102
57
|
*/
|
|
103
58
|
beginTurn(input: BeginTurnInput): Promise<TurnHandle>;
|
|
104
59
|
/**
|
|
105
|
-
* Commit a
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
* a no-op on the happy path.
|
|
60
|
+
* Commit a turn — verifies the addon saved the cache file, records it as
|
|
61
|
+
* established, and (for auto-cache turns) renames the addon's pre-response
|
|
62
|
+
* file to the post-response path. Flips the turn's internal `committed`
|
|
63
|
+
* flag so the deferred `rollback` becomes a no-op.
|
|
110
64
|
*/
|
|
111
65
|
commitTurn(turn: TurnHandle, result: CommitResult): Promise<void>;
|
|
112
66
|
/**
|
|
113
67
|
* Roll back an in-flight turn — atomically deletes the on-disk cache
|
|
114
|
-
* file,
|
|
115
|
-
* `
|
|
116
|
-
* removes orphaned marker metadata.
|
|
117
|
-
* already been committed or rolled back is a
|
|
118
|
-
* calls. Handlers register this via
|
|
119
|
-
* runs regardless of how the handler exits
|
|
120
|
-
* itself via `commitTurn`).
|
|
68
|
+
* file, drops the addon's copy of the conversation through
|
|
69
|
+
* `options.discardCache`, clears the in-memory `initializedCaches` entry,
|
|
70
|
+
* releases the active-path ref, and removes orphaned marker metadata.
|
|
71
|
+
* Idempotent: a turn that has already been committed or rolled back is a
|
|
72
|
+
* no-op on subsequent calls. Handlers register this via
|
|
73
|
+
* `ctx.scope.defer(...)` so it runs regardless of how the handler exits
|
|
74
|
+
* (success branch removes itself via `commitTurn`).
|
|
121
75
|
*/
|
|
122
76
|
rollback(turn: TurnHandle): Promise<void>;
|
|
123
77
|
/**
|
|
124
78
|
* Non-destructive counterpart of `rollback`: releases locks and refs but
|
|
125
|
-
* keeps the committed disk cache
|
|
126
|
-
*
|
|
79
|
+
* keeps the committed disk cache for a retry. A cache this same turn
|
|
80
|
+
* created rolls back instead.
|
|
127
81
|
*/
|
|
128
82
|
releaseTurn(turn: TurnHandle): Promise<void>;
|
|
129
|
-
/**
|
|
130
|
-
* Forget the saved-message count for the turn's path, in memory and in the
|
|
131
|
-
* sidecar, without unlinking the cache file or clearing the init flag. Used
|
|
132
|
-
* when `decideCachedHistorySlice` detects a stale boundary
|
|
133
|
-
* (`clearStaleCount: true`) — the next turn re-sends the full history
|
|
134
|
-
* but the cache itself is still usable.
|
|
135
|
-
*/
|
|
136
|
-
dropStaleSavedCount(turn: TurnHandle): Promise<void>;
|
|
137
83
|
}
|
|
138
84
|
/**
|
|
139
85
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
140
86
|
* scope. `options.logger` is the per-instance logger the session emits
|
|
141
87
|
* through (typically `withRequestContext(getEngineLogger(), ctx)`);
|
|
142
|
-
* falls back to the module-scoped logger when omitted.
|
|
88
|
+
* falls back to the module-scoped logger when omitted. `options.discardCache`
|
|
89
|
+
* drops the addon's copy of a conversation whose cache the session rolls
|
|
90
|
+
* back; without it the addon writes that conversation to the deleted path at
|
|
91
|
+
* its next key switch or unload.
|
|
143
92
|
*/
|
|
144
93
|
export declare function createKvCacheSession(modelId: string, options?: {
|
|
145
94
|
logger?: Logger;
|
|
95
|
+
discardCache?: (cachePath: string) => Promise<void>;
|
|
146
96
|
}): KvCacheSession;
|
|
147
97
|
/**
|
|
148
98
|
* Atomically delete every layer of KV-cache state for a
|
|
@@ -161,10 +111,8 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
161
111
|
* Layers cleared, in order:
|
|
162
112
|
* 1. On-disk: `deleteCache(...)` removes the matching directory
|
|
163
113
|
* tree (or wipes and recreates the root for `all: true`).
|
|
164
|
-
* 2. `
|
|
165
|
-
*
|
|
166
|
-
* 3. `initializedCaches`: prefix-cleanup by the same removed
|
|
167
|
-
* directory, since it is keyed by the resolved cache path.
|
|
114
|
+
* 2. `initializedCaches`: prefix-cleanup by the removed directory,
|
|
115
|
+
* since it is keyed by the resolved cache path.
|
|
168
116
|
*
|
|
169
117
|
* Concurrency with in-flight turns: this delete does not take the
|
|
170
118
|
* per-cache-path write locks, so it races any turn holding a `TurnHandle`
|
|
@@ -173,9 +121,8 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
173
121
|
* `Map.delete` / `Set.delete` no-op on absent keys), but the layers are
|
|
174
122
|
* cleared without a lock, so an interleaving that lands the delete's
|
|
175
123
|
* in-memory cleanup after a concurrent turn has already renamed/committed
|
|
176
|
-
* its file splits state: the file stays on disk while its
|
|
177
|
-
*
|
|
178
|
-
* `savedCount=0`. Callers must not delete a key that is in active use.
|
|
124
|
+
* its file splits state: the file stays on disk while its init flag is
|
|
125
|
+
* cleared. Callers must not delete a key that is in active use.
|
|
179
126
|
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
180
127
|
* and clears each one under the cache-state lock.
|
|
181
128
|
*/
|
|
@@ -197,11 +144,6 @@ export declare function deleteKvCacheState(target: {
|
|
|
197
144
|
* @internal
|
|
198
145
|
*/
|
|
199
146
|
export declare const __kvCacheSessionTestHooks: {
|
|
200
|
-
getSavedCount(cachePath: string): number | undefined;
|
|
201
|
-
setSavedCountForTest(cachePath: string, count: number): void;
|
|
202
|
-
getToolBlockCachedForTest(cachePath: string): boolean;
|
|
203
|
-
getPrefixSidecarPathForTest(cachePath: string): string;
|
|
204
|
-
readPrefixSidecarForTest(cachePath: string): Promise<CachedPrefix | null>;
|
|
205
147
|
hasInitializedPath(cachePath: string): boolean;
|
|
206
148
|
getActivePathCountForTest(cachePath: string): number;
|
|
207
149
|
getLastAutoCacheSweepMsForTest(): number;
|