@qvac/inference 0.21.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +30 -1
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +4 -1
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +5 -4
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +80 -28
- package/dist/handlers/get-loaded-model-info.js +1 -2
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +3 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/logger.js +1 -1
- package/dist/logging/utils.js +1 -1
- package/dist/models/registry/models.d.ts +563 -816
- package/dist/models/registry/models.js +2873 -3110
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +203 -2
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
- package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
- package/dist/plugins/builtin/index.d.ts +2 -0
- package/dist/plugins/builtin/index.js +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
- package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
- package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +10 -25
- package/dist/plugins/ops/load-model.js +21 -16
- package/dist/plugins/ops/transcribe.js +17 -1
- package/dist/registry.js +1 -0
- package/dist/resources/model-fit/assess.d.ts +15 -23
- package/dist/resources/model-fit/assess.js +207 -271
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
- package/dist/resources/model-fit/floor.d.ts +10 -10
- package/dist/resources/model-fit/floor.js +13 -13
- package/dist/resources/model-fit/kv-cache.d.ts +23 -0
- package/dist/resources/model-fit/kv-cache.js +85 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
- package/dist/resources/model-fit/platform.d.ts +4 -4
- package/dist/resources/model-fit/platform.js +6 -8
- package/dist/resources/model-fit/types.d.ts +3 -117
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/model-config-utils.js +6 -1
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/request-context.js +8 -0
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +100 -43
- package/dist/schemas/assess-model-fit.js +66 -29
- package/dist/schemas/audio-gen.d.ts +186 -18
- package/dist/schemas/audio-gen.js +180 -48
- package/dist/schemas/batch-completion-stream.d.ts +202 -34
- package/dist/schemas/bci-config.d.ts +2 -0
- package/dist/schemas/bci.d.ts +23 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +1083 -158
- package/dist/schemas/common.js +5 -2
- package/dist/schemas/completion-stream.d.ts +232 -37
- package/dist/schemas/completion-stream.js +64 -20
- package/dist/schemas/config.d.ts +477 -12
- package/dist/schemas/config.js +21 -1
- package/dist/schemas/decision.d.ts +602 -0
- package/dist/schemas/decision.js +205 -0
- package/dist/schemas/download-asset.d.ts +7 -3
- package/dist/schemas/engine-addon-map.d.ts +7 -0
- package/dist/schemas/engine-addon-map.js +23 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.js +2 -0
- package/dist/schemas/llamacpp-config.d.ts +42 -2
- package/dist/schemas/llamacpp-config.js +27 -0
- package/dist/schemas/load-model.d.ts +5247 -1309
- package/dist/schemas/load-model.js +88 -9
- package/dist/schemas/model-config-schemas.d.ts +233 -8
- package/dist/schemas/model-config-schemas.js +4 -1
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +2 -1
- package/dist/schemas/model-resource-profile.d.ts +3 -4
- package/dist/schemas/model-resource-profile.js +1 -4
- package/dist/schemas/model-src-utils.d.ts +10 -2
- package/dist/schemas/model-types.d.ts +10 -1
- package/dist/schemas/model-types.js +3 -0
- package/dist/schemas/ocr.d.ts +74 -0
- package/dist/schemas/ocr.js +42 -0
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/registry.d.ts +26 -0
- package/dist/schemas/registry.js +3 -0
- package/dist/schemas/reload-config.d.ts +10 -2
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +26 -0
- package/dist/schemas/sdcpp-config.js +8 -7
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +402 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +15 -0
- package/dist/schemas/transcription-config.js +38 -3
- package/dist/schemas/transcription.d.ts +46 -4
- package/dist/schemas/transcription.js +51 -4
- package/dist/schemas/translation-config.d.ts +30 -10
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/mcp-adapter.js +12 -8
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +1 -0
- package/package.json +41 -32
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
|
@@ -6,22 +6,19 @@ import { isMobile } from '../../../../runtime/state.js';
|
|
|
6
6
|
import { getKVCacheDir } from '../../../../utils/index.js';
|
|
7
7
|
import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
|
|
8
8
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
9
|
-
import { z } from 'zod';
|
|
10
9
|
// Used by cross-model paths that have no `RequestContext` (e.g.
|
|
11
10
|
// `deleteKvCacheState`). Per-session call sites receive a logger from
|
|
12
11
|
// the caller — typically `withRequestContext(...)`.
|
|
13
12
|
const moduleLogger = getEngineLogger();
|
|
14
13
|
/**
|
|
15
|
-
* Coordinates
|
|
14
|
+
* Coordinates four KV-cache state layers:
|
|
16
15
|
*
|
|
17
|
-
* 1. `
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* 5. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
24
|
-
* 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
16
|
+
* 1. `initializedCaches` — caches this process has established.
|
|
17
|
+
* 2. On-disk `.bin` files written by the addon. Each one records the tokens
|
|
18
|
+
* it holds, and the addon reuses the longest prefix the next prompt
|
|
19
|
+
* shares with it, so nothing here tracks what a file covers.
|
|
20
|
+
* 3. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
21
|
+
* 4. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
25
22
|
*
|
|
26
23
|
* Every turn must finish through `commitTurn`, `rollback`, or the
|
|
27
24
|
* non-destructive `releaseTurn` so all inference state stays aligned,
|
|
@@ -30,100 +27,6 @@ const moduleLogger = getEngineLogger();
|
|
|
30
27
|
*/
|
|
31
28
|
// ----- module-scoped state. The session is the single mutation point
|
|
32
29
|
// for the in-memory KV-cache bookkeeping. -----
|
|
33
|
-
/** What the kv-cache file at a given path is known to hold. */
|
|
34
|
-
const cachedPrefixSchema = z.object({
|
|
35
|
-
/** Number of chat messages the file on disk is known to cover. */
|
|
36
|
-
messages: z.number().int().nonnegative(),
|
|
37
|
-
/**
|
|
38
|
-
* Whether a static tool block was rendered into that prefix. Tracked
|
|
39
|
-
* rather than inferred from `messages`, because a committed turn is not
|
|
40
|
-
* proof that its tool block reached the model: the addon drops tools and
|
|
41
|
-
* still returns a usable prompt when the chat template rejects them.
|
|
42
|
-
*/
|
|
43
|
-
toolBlock: z.boolean()
|
|
44
|
-
});
|
|
45
|
-
// The sidecar is written after the addon has already saved the `.bin`, so a
|
|
46
|
-
// crash between the two leaves a boundary that describes an older file. The
|
|
47
|
-
// `.bin` size is a function of the tokens it holds: it is unchanged when the
|
|
48
|
-
// addon re-saves the same state (cancel rewind, key switch) and larger once
|
|
49
|
-
// another turn landed in it, which is the case the fingerprint must catch.
|
|
50
|
-
const persistedPrefixSchema = cachedPrefixSchema.extend({
|
|
51
|
-
binSize: z.number().int().nonnegative()
|
|
52
|
-
});
|
|
53
|
-
/**
|
|
54
|
-
* What the kv-cache file on disk is known to cover, keyed by cache path.
|
|
55
|
-
* Written by `commitTurn`, deleted by `rollback`, `delete`,
|
|
56
|
-
* `dropStaleSavedCount` and the orphan clear in `beginTurn`. The same
|
|
57
|
-
* INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
|
|
58
|
-
* present only when the corresponding `.bin` file is considered trustworthy,
|
|
59
|
-
* so any turn that leaves the file holding something the boundary does not
|
|
60
|
-
* describe has to remove it.
|
|
61
|
-
*
|
|
62
|
-
* This map outranks the sidecar. Both are written by the same commit, so an
|
|
63
|
-
* entry here is never the older copy, and `restorePersistedCache` reads the
|
|
64
|
-
* file only to repopulate an empty map after a restart.
|
|
65
|
-
*/
|
|
66
|
-
const cachedPrefixes = new Map();
|
|
67
|
-
const PREFIX_SIDECAR_SUFFIX = '.meta.json';
|
|
68
|
-
function prefixSidecarPath(cachePath) {
|
|
69
|
-
return `${cachePath}${PREFIX_SIDECAR_SUFFIX}`;
|
|
70
|
-
}
|
|
71
|
-
async function writePrefixSidecar(cachePath, prefix, logger) {
|
|
72
|
-
try {
|
|
73
|
-
const { size } = await fsPromises.stat(cachePath);
|
|
74
|
-
await fsPromises.writeFile(prefixSidecarPath(cachePath), JSON.stringify({ ...prefix, binSize: size }));
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
// Only the file goes: a sidecar left from the previous commit describes a
|
|
78
|
-
// shorter `.bin` than the one now on disk. The in-memory boundary is the
|
|
79
|
-
// one this process keeps slicing against and stays.
|
|
80
|
-
await removePrefixSidecar(cachePath);
|
|
81
|
-
logger.warn(`[kv-cache] Failed to persist saved-message boundary; this process stays warm but a restart will start the cache cold. path=${cachePath} error=${error instanceof Error ? error.message : String(error)}`);
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
// Anything unreadable, malformed, or describing a `.bin` of a different size
|
|
85
|
-
// is discarded: the cache is then used from a cold boundary, which is the
|
|
86
|
-
// pre-sidecar behaviour. Only ever consulted for a path `cachedPrefixes` has
|
|
87
|
-
// no entry for, so it can never contradict a live boundary. Never throws: the
|
|
88
|
-
// caller holds the cache-path write lock and a `.bin` can vanish under it
|
|
89
|
-
// (`deleteKvCacheState` takes no locks).
|
|
90
|
-
async function readPrefixSidecar(cachePath) {
|
|
91
|
-
let parsed;
|
|
92
|
-
let binSize;
|
|
93
|
-
try {
|
|
94
|
-
const raw = await fsPromises.readFile(prefixSidecarPath(cachePath), 'utf8');
|
|
95
|
-
const result = persistedPrefixSchema.safeParse(JSON.parse(raw));
|
|
96
|
-
if (!result.success)
|
|
97
|
-
return null;
|
|
98
|
-
parsed = result.data;
|
|
99
|
-
binSize = (await fsPromises.stat(cachePath)).size;
|
|
100
|
-
}
|
|
101
|
-
catch {
|
|
102
|
-
return null;
|
|
103
|
-
}
|
|
104
|
-
if (binSize !== parsed.binSize) {
|
|
105
|
-
// The `.bin` changed after this boundary was recorded, so the file is
|
|
106
|
-
// stale whatever wrote it. Only the file: the caller guarantees the map
|
|
107
|
-
// holds nothing for this path.
|
|
108
|
-
await removePrefixSidecar(cachePath);
|
|
109
|
-
return null;
|
|
110
|
-
}
|
|
111
|
-
return { messages: parsed.messages, toolBlock: parsed.toolBlock };
|
|
112
|
-
}
|
|
113
|
-
/** Drops the persisted boundary, leaving whatever is in memory alone. */
|
|
114
|
-
async function removePrefixSidecar(cachePath) {
|
|
115
|
-
try {
|
|
116
|
-
await fsPromises.unlink(prefixSidecarPath(cachePath));
|
|
117
|
-
}
|
|
118
|
-
catch {
|
|
119
|
-
// No sidecar for this path.
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
/** Drops the boundary everywhere: the cache it describes is gone or untrusted. */
|
|
123
|
-
async function forgetPrefix(cachePath) {
|
|
124
|
-
cachedPrefixes.delete(cachePath);
|
|
125
|
-
await removePrefixSidecar(cachePath);
|
|
126
|
-
}
|
|
127
30
|
/**
|
|
128
31
|
* In-memory registry of caches established this session, recorded once a save
|
|
129
32
|
* is verified. The addon defers disk writes, so the absence of a `.bin` file on
|
|
@@ -154,7 +57,7 @@ class CacheLockAbortError extends Error {
|
|
|
154
57
|
// "session"), which name the SAME file on case-insensitive filesystems (default
|
|
155
58
|
// macOS/Windows), serialise on one lock and can't interleave writes. Over-locks
|
|
156
59
|
// case-only variants on case-sensitive filesystems, which is safe. NOTE: the
|
|
157
|
-
// other per-path bookkeeping (activeCachePaths,
|
|
60
|
+
// other per-path bookkeeping (activeCachePaths, initializedCaches, retention)
|
|
158
61
|
// stays case-sensitive, so case-only key variants are otherwise unsupported —
|
|
159
62
|
// see the KV-cache system doc.
|
|
160
63
|
function lockKeyFor(cachePath) {
|
|
@@ -335,20 +238,24 @@ async function deleteInactiveAutoCaches() {
|
|
|
335
238
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
336
239
|
* scope. `options.logger` is the per-instance logger the session emits
|
|
337
240
|
* through (typically `withRequestContext(getEngineLogger(), ctx)`);
|
|
338
|
-
* falls back to the module-scoped logger when omitted.
|
|
241
|
+
* falls back to the module-scoped logger when omitted. `options.discardCache`
|
|
242
|
+
* drops the addon's copy of a conversation whose cache the session rolls
|
|
243
|
+
* back; without it the addon writes that conversation to the deleted path at
|
|
244
|
+
* its next key switch or unload.
|
|
339
245
|
*/
|
|
340
246
|
export function createKvCacheSession(modelId, options) {
|
|
341
247
|
const logger = options?.logger ?? moduleLogger;
|
|
248
|
+
const discardCache = options?.discardCache;
|
|
342
249
|
// Per-session map: each `TurnHandle` carries an opaque entry here. A
|
|
343
250
|
// WeakMap so handles drop their state once the handler scope releases
|
|
344
251
|
// the reference; the module-scoped maps above survive.
|
|
345
252
|
const turnState = new WeakMap();
|
|
346
253
|
function makeHandle(cachePath, autoCacheKey, releaseWriteLock = () => { }, signal) {
|
|
347
|
-
const cached = cachedPrefixes.get(cachePath);
|
|
348
254
|
const handle = {
|
|
349
255
|
cachePath,
|
|
350
|
-
|
|
351
|
-
|
|
256
|
+
get createdCache() {
|
|
257
|
+
return turnState.get(handle)?.createdByThisTurn ?? false;
|
|
258
|
+
}
|
|
352
259
|
};
|
|
353
260
|
turnState.set(handle, {
|
|
354
261
|
cachePath,
|
|
@@ -389,14 +296,10 @@ export function createKvCacheSession(modelId, options) {
|
|
|
389
296
|
// In-memory registry check first — the addon defers disk writes, so a
|
|
390
297
|
// just-saved cache may not yet exist on disk. If the in-memory flag isn't
|
|
391
298
|
// set, fall back to a filesystem probe so caches surviving across process
|
|
392
|
-
// restarts still hit the reuse path.
|
|
393
|
-
// it snapshots the restored boundary.
|
|
299
|
+
// restarts still hit the reuse path.
|
|
394
300
|
let exists = initializedCaches.has(cachePath);
|
|
395
301
|
if (!exists)
|
|
396
|
-
exists = await
|
|
397
|
-
// A boundary without its `.bin` describes a cache that no longer exists.
|
|
398
|
-
if (!exists)
|
|
399
|
-
await forgetPrefix(cachePath);
|
|
302
|
+
exists = await adoptPersistedCache(cachePath);
|
|
400
303
|
const handle = makeHandle(cachePath, undefined, releaseWriteLock, input.signal);
|
|
401
304
|
try {
|
|
402
305
|
logCacheStatus(input.customKey, exists);
|
|
@@ -511,17 +414,13 @@ export function createKvCacheSession(modelId, options) {
|
|
|
511
414
|
if (state.committed || state.rolledBack)
|
|
512
415
|
return;
|
|
513
416
|
if (result.kind === 'static') {
|
|
514
|
-
// Custom-key path: the addon wrote the new cache state
|
|
515
|
-
//
|
|
516
|
-
|
|
517
|
-
// `verifySaveAndRecord`) and record the new boundary.
|
|
518
|
-
const prefix = { messages: result.messageCount, toolBlock: result.toolBlockCached };
|
|
519
|
-
const ok = await verifySaveAndRecord(state.cachePath, prefix);
|
|
417
|
+
// Custom-key path: the addon's save wrote the new cache state at the
|
|
418
|
+
// same path. Verify the file persisted.
|
|
419
|
+
const ok = await verifySaveAndRecord(state.cachePath);
|
|
520
420
|
if (!ok) {
|
|
521
421
|
await runRollback(state);
|
|
522
422
|
return;
|
|
523
423
|
}
|
|
524
|
-
await writePrefixSidecar(state.cachePath, prefix, logger);
|
|
525
424
|
state.committed = true;
|
|
526
425
|
releaseCachePath(state.cachePath);
|
|
527
426
|
state.releaseWriteLock();
|
|
@@ -531,13 +430,14 @@ export function createKvCacheSession(modelId, options) {
|
|
|
531
430
|
const sourceCacheKey = state.autoCacheKey;
|
|
532
431
|
const targetCacheKey = path.basename(path.dirname(path.dirname(result.targetCachePath)));
|
|
533
432
|
// Hold the target's lock across the rename AND commit verification, so a peer
|
|
534
|
-
// resolving to the same file can't observe it before
|
|
535
|
-
// published. Abortable: a cancel while waiting for the lock rejects here.
|
|
433
|
+
// resolving to the same file can't observe it before it is recorded. Abortable: a cancel while waiting for the lock rejects here.
|
|
536
434
|
const releaseTargetLock = await acquireCachePathWriteLock(result.targetCachePath, state.signal);
|
|
537
435
|
try {
|
|
538
|
-
// A cancel that landed while we waited for the target lock must not
|
|
436
|
+
// A cancel that landed while we waited for the target lock must not
|
|
437
|
+
// commit. The file holds what the addon kept, so it stays unless this
|
|
438
|
+
// turn created it.
|
|
539
439
|
if (state.signal?.aborted) {
|
|
540
|
-
await
|
|
440
|
+
await runRelease(state);
|
|
541
441
|
return;
|
|
542
442
|
}
|
|
543
443
|
const renamed = await withCacheStateLock(async () => {
|
|
@@ -564,7 +464,6 @@ export function createKvCacheSession(modelId, options) {
|
|
|
564
464
|
releaseCachePath(sourceCachePath);
|
|
565
465
|
state.cachePath = result.targetCachePath;
|
|
566
466
|
state.autoCacheKey = targetCacheKey;
|
|
567
|
-
await forgetPrefix(sourceCachePath);
|
|
568
467
|
await pruneEmptyCacheDirectories(sourceCachePath, snapshotActivePaths());
|
|
569
468
|
if (sourceCacheKey !== undefined) {
|
|
570
469
|
await removeAutoCacheMarkerIfMissing(sourceCacheKey);
|
|
@@ -579,10 +478,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
579
478
|
await runRollback(state);
|
|
580
479
|
return;
|
|
581
480
|
}
|
|
582
|
-
const ok = await verifySaveAndRecord(result.targetCachePath
|
|
583
|
-
messages: result.messageCount,
|
|
584
|
-
toolBlock: result.toolBlockCached
|
|
585
|
-
});
|
|
481
|
+
const ok = await verifySaveAndRecord(result.targetCachePath);
|
|
586
482
|
if (!ok) {
|
|
587
483
|
// Rename succeeded but the file isn't where we expected. Roll back via
|
|
588
484
|
// the target path instead of the (now-empty) source.
|
|
@@ -615,6 +511,9 @@ export function createKvCacheSession(modelId, options) {
|
|
|
615
511
|
return;
|
|
616
512
|
if (state.committed || state.rolledBack)
|
|
617
513
|
return;
|
|
514
|
+
await runRelease(state);
|
|
515
|
+
}
|
|
516
|
+
async function runRelease(state) {
|
|
618
517
|
// Nothing committed exists to keep, so a failed first turn is destructive.
|
|
619
518
|
if (state.createdByThisTurn) {
|
|
620
519
|
await runRollback(state);
|
|
@@ -641,7 +540,14 @@ export function createKvCacheSession(modelId, options) {
|
|
|
641
540
|
logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
|
|
642
541
|
}
|
|
643
542
|
}
|
|
644
|
-
|
|
543
|
+
if (discardCache) {
|
|
544
|
+
try {
|
|
545
|
+
await discardCache(state.cachePath);
|
|
546
|
+
}
|
|
547
|
+
catch (discardError) {
|
|
548
|
+
logger.warn(`[kv-cache] Failed to discard the addon's copy of a rolled-back cache; a later unload may write it back. path=${state.cachePath} error=${discardError instanceof Error ? discardError.message : String(discardError)}`);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
645
551
|
// Release before pruning so an empty parent can go; a sibling still holding
|
|
646
552
|
// the path keeps it in the active snapshot and protects the directory.
|
|
647
553
|
releaseCachePath(state.cachePath);
|
|
@@ -655,18 +561,11 @@ export function createKvCacheSession(modelId, options) {
|
|
|
655
561
|
if (state.autoCacheKey !== undefined)
|
|
656
562
|
scheduleAutoCacheSweep(logger);
|
|
657
563
|
}
|
|
658
|
-
async function dropStaleSavedCount(turn) {
|
|
659
|
-
const state = turnState.get(turn);
|
|
660
|
-
if (!state)
|
|
661
|
-
return;
|
|
662
|
-
await forgetPrefix(state.cachePath);
|
|
663
|
-
}
|
|
664
564
|
return {
|
|
665
565
|
beginTurn,
|
|
666
566
|
commitTurn,
|
|
667
567
|
rollback,
|
|
668
|
-
releaseTurn
|
|
669
|
-
dropStaleSavedCount
|
|
568
|
+
releaseTurn
|
|
670
569
|
};
|
|
671
570
|
}
|
|
672
571
|
// ----- module-level administrative API -----
|
|
@@ -687,10 +586,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
687
586
|
* Layers cleared, in order:
|
|
688
587
|
* 1. On-disk: `deleteCache(...)` removes the matching directory
|
|
689
588
|
* tree (or wipes and recreates the root for `all: true`).
|
|
690
|
-
* 2. `
|
|
691
|
-
*
|
|
692
|
-
* 3. `initializedCaches`: prefix-cleanup by the same removed
|
|
693
|
-
* directory, since it is keyed by the resolved cache path.
|
|
589
|
+
* 2. `initializedCaches`: prefix-cleanup by the removed directory,
|
|
590
|
+
* since it is keyed by the resolved cache path.
|
|
694
591
|
*
|
|
695
592
|
* Concurrency with in-flight turns: this delete does not take the
|
|
696
593
|
* per-cache-path write locks, so it races any turn holding a `TurnHandle`
|
|
@@ -699,9 +596,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
699
596
|
* `Map.delete` / `Set.delete` no-op on absent keys), but the layers are
|
|
700
597
|
* cleared without a lock, so an interleaving that lands the delete's
|
|
701
598
|
* in-memory cleanup after a concurrent turn has already renamed/committed
|
|
702
|
-
* its file splits state: the file stays on disk while its
|
|
703
|
-
*
|
|
704
|
-
* `savedCount=0`. Callers must not delete a key that is in active use.
|
|
599
|
+
* its file splits state: the file stays on disk while its init flag is
|
|
600
|
+
* cleared. Callers must not delete a key that is in active use.
|
|
705
601
|
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
706
602
|
* and clears each one under the cache-state lock.
|
|
707
603
|
*/
|
|
@@ -710,7 +606,6 @@ export async function deleteKvCacheState(target) {
|
|
|
710
606
|
return deleteInactiveAutoCaches();
|
|
711
607
|
if ('all' in target) {
|
|
712
608
|
const removed = await deleteCacheUtil({ all: true });
|
|
713
|
-
cachedPrefixes.clear();
|
|
714
609
|
initializedCaches.clear();
|
|
715
610
|
// `removed` is the kv-cache root dir; surfaces it for ops
|
|
716
611
|
// visibility but isn't part of the contract.
|
|
@@ -728,50 +623,40 @@ export async function deleteKvCacheState(target) {
|
|
|
728
623
|
// onto the unrelated top-level auto marker.
|
|
729
624
|
await removeAutoCacheMarker(path.relative(getKVCacheDir(), removedPath));
|
|
730
625
|
}
|
|
731
|
-
//
|
|
732
|
-
//
|
|
733
|
-
//
|
|
734
|
-
// or sits beneath it must go.
|
|
735
|
-
clearCachedMessageCountsByPrefix(removedPath, path.sep);
|
|
736
|
-
// initializedCaches is keyed by the resolved cachePath too, so clear it by
|
|
737
|
-
// the same removed-directory prefix as cachedPrefixes above.
|
|
626
|
+
// The on-disk directory tree is `{kvCacheRoot}/{kvCacheKey}[/{modelId}]/`,
|
|
627
|
+
// so every entry whose path is the removed directory itself or sits beneath
|
|
628
|
+
// it must go.
|
|
738
629
|
clearInitializedCachesByPrefix(removedPath, path.sep);
|
|
739
630
|
}
|
|
740
631
|
// ----- private helpers -----
|
|
741
632
|
/**
|
|
742
|
-
* Verify the
|
|
743
|
-
*
|
|
744
|
-
*
|
|
745
|
-
* history rather than slicing against a stale `savedCount`.
|
|
746
|
-
*
|
|
747
|
-
* TODO: once the addon surfaces save failures (e.g. throws
|
|
748
|
-
* `UnableToSaveSessionFile` when `llama_state_save_file` returns
|
|
749
|
-
* false), drop the `access()` probe and wrap the `model.run()` call in
|
|
750
|
-
* a real try/catch that forwards the error.
|
|
633
|
+
* Verify the cache file exists before recording it as established. The file
|
|
634
|
+
* is the evidence because `saveCache` rejects both when a write fails and when
|
|
635
|
+
* the addon kept nothing to write.
|
|
751
636
|
*/
|
|
752
|
-
async function verifySaveAndRecord(cachePath
|
|
637
|
+
async function verifySaveAndRecord(cachePath) {
|
|
753
638
|
try {
|
|
754
639
|
await fsPromises.access(cachePath);
|
|
755
|
-
cachedPrefixes.set(cachePath, prefix);
|
|
756
640
|
initializedCaches.add(cachePath);
|
|
757
641
|
return true;
|
|
758
642
|
}
|
|
759
643
|
catch (err) {
|
|
760
|
-
cachedPrefixes.delete(cachePath);
|
|
761
644
|
logCacheSaveError(cachePath, err);
|
|
762
645
|
return false;
|
|
763
646
|
}
|
|
764
647
|
}
|
|
765
648
|
/**
|
|
766
|
-
* Adopt a `.bin` left by an earlier process run
|
|
767
|
-
*
|
|
768
|
-
*
|
|
769
|
-
* An auto-rename commits its target into `cachedPrefixes` without an init
|
|
770
|
-
* flag, so this can run for a path the map already knows. The map wins there
|
|
771
|
-
* and the sidecar is left unread, which keeps the two from ever having to be
|
|
772
|
-
* reconciled.
|
|
649
|
+
* Adopt a `.bin` left by an earlier process run. Returns whether the file
|
|
650
|
+
* exists. Earlier releases kept a `<bin>.meta.json` boundary sidecar beside
|
|
651
|
+
* it; nothing reads one now, so it goes on first contact.
|
|
773
652
|
*/
|
|
774
|
-
async function
|
|
653
|
+
async function adoptPersistedCache(cachePath) {
|
|
654
|
+
try {
|
|
655
|
+
await fsPromises.unlink(`${cachePath}.meta.json`);
|
|
656
|
+
}
|
|
657
|
+
catch {
|
|
658
|
+
// No sidecar for this path.
|
|
659
|
+
}
|
|
775
660
|
try {
|
|
776
661
|
await fsPromises.access(cachePath);
|
|
777
662
|
}
|
|
@@ -779,28 +664,8 @@ async function restorePersistedCache(cachePath) {
|
|
|
779
664
|
return false;
|
|
780
665
|
}
|
|
781
666
|
initializedCaches.add(cachePath);
|
|
782
|
-
if (cachedPrefixes.has(cachePath))
|
|
783
|
-
return true;
|
|
784
|
-
const prefix = await readPrefixSidecar(cachePath);
|
|
785
|
-
if (prefix !== null)
|
|
786
|
-
cachedPrefixes.set(cachePath, prefix);
|
|
787
667
|
return true;
|
|
788
668
|
}
|
|
789
|
-
function clearCachedMessageCountsByPrefix(prefix, sep) {
|
|
790
|
-
if (!prefix) {
|
|
791
|
-
cachedPrefixes.clear();
|
|
792
|
-
return;
|
|
793
|
-
}
|
|
794
|
-
for (const key of cachedPrefixes.keys()) {
|
|
795
|
-
if (key === prefix) {
|
|
796
|
-
cachedPrefixes.delete(key);
|
|
797
|
-
continue;
|
|
798
|
-
}
|
|
799
|
-
if (!key.startsWith(prefix + sep))
|
|
800
|
-
continue;
|
|
801
|
-
cachedPrefixes.delete(key);
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
669
|
function clearInitializedCachesByPrefix(prefix, sep) {
|
|
805
670
|
if (!prefix) {
|
|
806
671
|
initializedCaches.clear();
|
|
@@ -822,21 +687,6 @@ function clearInitializedCachesByPrefix(prefix, sep) {
|
|
|
822
687
|
* @internal
|
|
823
688
|
*/
|
|
824
689
|
export const __kvCacheSessionTestHooks = {
|
|
825
|
-
getSavedCount(cachePath) {
|
|
826
|
-
return cachedPrefixes.get(cachePath)?.messages;
|
|
827
|
-
},
|
|
828
|
-
setSavedCountForTest(cachePath, count) {
|
|
829
|
-
cachedPrefixes.set(cachePath, { messages: count, toolBlock: false });
|
|
830
|
-
},
|
|
831
|
-
getToolBlockCachedForTest(cachePath) {
|
|
832
|
-
return cachedPrefixes.get(cachePath)?.toolBlock ?? false;
|
|
833
|
-
},
|
|
834
|
-
getPrefixSidecarPathForTest(cachePath) {
|
|
835
|
-
return prefixSidecarPath(cachePath);
|
|
836
|
-
},
|
|
837
|
-
readPrefixSidecarForTest(cachePath) {
|
|
838
|
-
return readPrefixSidecar(cachePath);
|
|
839
|
-
},
|
|
840
690
|
hasInitializedPath(cachePath) {
|
|
841
691
|
return initializedCaches.has(cachePath);
|
|
842
692
|
},
|
|
@@ -850,7 +700,6 @@ export const __kvCacheSessionTestHooks = {
|
|
|
850
700
|
lastAutoCacheSweepMs = value;
|
|
851
701
|
},
|
|
852
702
|
resetForTest() {
|
|
853
|
-
cachedPrefixes.clear();
|
|
854
703
|
initializedCaches.clear();
|
|
855
704
|
activeCachePaths.clear();
|
|
856
705
|
lastAutoCacheSweepMs = 0;
|
|
@@ -1,37 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pure decision
|
|
3
|
-
* `completion-stream.ts` (via the slice computed from
|
|
4
|
-
* `TurnHandle.savedCount`).
|
|
2
|
+
* Pure commit decision for kv-cache turns used by `completion-stream.ts`.
|
|
5
3
|
*
|
|
6
4
|
* This module intentionally has **no** `bare-*` imports so it can be
|
|
7
|
-
* exercised directly from unit tests running under
|
|
5
|
+
* exercised directly from unit tests running under Node without
|
|
8
6
|
* pulling in the Bare runtime (which is not available in that
|
|
9
7
|
* environment).
|
|
10
|
-
*
|
|
11
|
-
* Cancel detection flows through the per-request `AbortSignal` from
|
|
12
|
-
* `RequestRegistry`; the `cachedMessageCounts` map lives in
|
|
13
|
-
* `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
|
|
14
|
-
* layers (saved counts, init flags, on-disk files). Only the pure
|
|
15
|
-
* slice-decision helper remains here.
|
|
16
8
|
*/
|
|
17
|
-
export interface HistoryMessage {
|
|
18
|
-
role: string;
|
|
19
|
-
content: string;
|
|
20
|
-
attachments?: {
|
|
21
|
-
path: string;
|
|
22
|
-
}[] | undefined;
|
|
23
|
-
}
|
|
24
|
-
export interface HistorySliceDecision {
|
|
25
|
-
/** Messages to send to the model on the next turn. */
|
|
26
|
-
messages: HistoryMessage[];
|
|
27
|
-
/**
|
|
28
|
-
* True when the decision path proves the current `savedCount` is stale
|
|
29
|
-
* and the caller should drop the cached entry (via
|
|
30
|
-
* `KvCacheSession.dropStaleSavedCount(turn)`) to avoid propagating the
|
|
31
|
-
* bad count to the next turn.
|
|
32
|
-
*/
|
|
33
|
-
clearStaleCount: boolean;
|
|
34
|
-
}
|
|
35
9
|
interface CacheCommitContext {
|
|
36
10
|
aborted: boolean;
|
|
37
11
|
producedTokens: boolean;
|
|
@@ -40,25 +14,4 @@ interface CacheCommitContext {
|
|
|
40
14
|
stoppedAtContextBoundary: boolean;
|
|
41
15
|
}
|
|
42
16
|
export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
|
|
43
|
-
/**
|
|
44
|
-
* Pure slice decision for `prepareMessagesForCache`.
|
|
45
|
-
*
|
|
46
|
-
* Mirrors the shape of the logic in `completion-stream.ts` but without
|
|
47
|
-
* calling `transformMessages` (which depends on `bare-fs` for
|
|
48
|
-
* attachment probing). Kept here so the decision can be unit-tested in
|
|
49
|
-
* isolation.
|
|
50
|
-
*
|
|
51
|
-
* A non-zero `savedCount` is itself the proof that a cache holds a rendered
|
|
52
|
-
* prefix: it is recorded only after a save is verified. It means the prefix
|
|
53
|
-
* covers that many messages, so slicing from it drops exactly what the cache
|
|
54
|
-
* supplies — the system message among them. Without a usable boundary the
|
|
55
|
-
* cache supplies nothing, so the whole history goes to the addon, system
|
|
56
|
-
* message included: anything held back would simply be missing from the
|
|
57
|
-
* prompt.
|
|
58
|
-
*
|
|
59
|
-
* The regression guard: a non-zero `savedCount` that slices the history down
|
|
60
|
-
* to an empty array is stale, and the caller resends the full history rather
|
|
61
|
-
* than handing the model an empty payload.
|
|
62
|
-
*/
|
|
63
|
-
export declare function decideCachedHistorySlice(savedCount: number, history: HistoryMessage[]): HistorySliceDecision;
|
|
64
17
|
export {};
|
|
@@ -1,18 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pure decision
|
|
3
|
-
* `completion-stream.ts` (via the slice computed from
|
|
4
|
-
* `TurnHandle.savedCount`).
|
|
2
|
+
* Pure commit decision for kv-cache turns used by `completion-stream.ts`.
|
|
5
3
|
*
|
|
6
4
|
* This module intentionally has **no** `bare-*` imports so it can be
|
|
7
|
-
* exercised directly from unit tests running under
|
|
5
|
+
* exercised directly from unit tests running under Node without
|
|
8
6
|
* pulling in the Bare runtime (which is not available in that
|
|
9
7
|
* environment).
|
|
10
|
-
*
|
|
11
|
-
* Cancel detection flows through the per-request `AbortSignal` from
|
|
12
|
-
* `RequestRegistry`; the `cachedMessageCounts` map lives in
|
|
13
|
-
* `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
|
|
14
|
-
* layers (saved counts, init flags, on-disk files). Only the pure
|
|
15
|
-
* slice-decision helper remains here.
|
|
16
8
|
*/
|
|
17
9
|
export function shouldCommitCachedTurn(context) {
|
|
18
10
|
const { aborted, producedTokens, generatedTokens, predict, stoppedAtContextBoundary } = context;
|
|
@@ -22,39 +14,3 @@ export function shouldCommitCachedTurn(context) {
|
|
|
22
14
|
generatedTokens >= predict;
|
|
23
15
|
return !aborted && producedTokens && !stoppedByBudget && !stoppedAtContextBoundary;
|
|
24
16
|
}
|
|
25
|
-
/**
|
|
26
|
-
* Pure slice decision for `prepareMessagesForCache`.
|
|
27
|
-
*
|
|
28
|
-
* Mirrors the shape of the logic in `completion-stream.ts` but without
|
|
29
|
-
* calling `transformMessages` (which depends on `bare-fs` for
|
|
30
|
-
* attachment probing). Kept here so the decision can be unit-tested in
|
|
31
|
-
* isolation.
|
|
32
|
-
*
|
|
33
|
-
* A non-zero `savedCount` is itself the proof that a cache holds a rendered
|
|
34
|
-
* prefix: it is recorded only after a save is verified. It means the prefix
|
|
35
|
-
* covers that many messages, so slicing from it drops exactly what the cache
|
|
36
|
-
* supplies — the system message among them. Without a usable boundary the
|
|
37
|
-
* cache supplies nothing, so the whole history goes to the addon, system
|
|
38
|
-
* message included: anything held back would simply be missing from the
|
|
39
|
-
* prompt.
|
|
40
|
-
*
|
|
41
|
-
* The regression guard: a non-zero `savedCount` that slices the history down
|
|
42
|
-
* to an empty array is stale, and the caller resends the full history rather
|
|
43
|
-
* than handing the model an empty payload.
|
|
44
|
-
*/
|
|
45
|
-
export function decideCachedHistorySlice(savedCount, history) {
|
|
46
|
-
const hasCachedPrefix = savedCount > 0 && history.length > 0;
|
|
47
|
-
const sliced = hasCachedPrefix && savedCount <= history.length ? history.slice(savedCount) : null;
|
|
48
|
-
// A non-null slice that is empty means the saved count is stale: the
|
|
49
|
-
// cached turn boundary is claiming the entire current history is
|
|
50
|
-
// already cached, which happens when a previous turn was cancelled
|
|
51
|
-
// mid-decode and still recorded `history.length + 1`. Treat it as a
|
|
52
|
-
// bad state and resend the full history.
|
|
53
|
-
const useSlice = sliced !== null && sliced.length > 0;
|
|
54
|
-
return {
|
|
55
|
-
messages: useSlice ? sliced : history,
|
|
56
|
-
// Only a boundary that was actually consulted can be stale. Absent a
|
|
57
|
-
// history to slice, the count is simply unused and left alone.
|
|
58
|
-
clearStaleCount: hasCachedPrefix && !useSlice
|
|
59
|
-
};
|
|
60
|
-
}
|