@qvac/inference 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -39
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +84 -0
- package/dist/schemas/assess-model-fit.js +72 -3
- package/dist/schemas/audio-gen.d.ts +482 -7
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +10 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +782 -19
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +14 -1
- package/dist/schemas/completion-event.js +4 -0
- package/dist/schemas/completion-stream.d.ts +22 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2604 -180
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +21 -11
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +30 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
|
@@ -6,27 +6,130 @@ import { isMobile } from '../../../../runtime/state.js';
|
|
|
6
6
|
import { getKVCacheDir } from '../../../../utils/index.js';
|
|
7
7
|
import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
|
|
8
8
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
9
|
+
import { z } from 'zod';
|
|
9
10
|
// Used by cross-model paths that have no `RequestContext` (e.g.
|
|
10
11
|
// `deleteKvCacheState`). Per-session call sites receive a logger from
|
|
11
12
|
// the caller — typically `withRequestContext(...)`.
|
|
12
13
|
const moduleLogger = getEngineLogger();
|
|
14
|
+
/**
|
|
15
|
+
* Coordinates six KV-cache state layers:
|
|
16
|
+
*
|
|
17
|
+
* 1. `cachedPrefixes` — saved message boundaries, and whether a tool
|
|
18
|
+
* block was rendered into them.
|
|
19
|
+
* 2. `initializedCaches` — caches this process has established.
|
|
20
|
+
* 3. On-disk `.bin` files written by the addon.
|
|
21
|
+
* 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
|
|
22
|
+
* boundary outlives the process; see `readPrefixSidecar`.
|
|
23
|
+
* 5. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
24
|
+
* 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
25
|
+
*
|
|
26
|
+
* Every turn must finish through `commitTurn`, `rollback`, or the
|
|
27
|
+
* non-destructive `releaseTurn` so all inference state stays aligned,
|
|
28
|
+
* the active-path ref is released, and marker metadata follows the
|
|
29
|
+
* cache directory lifecycle.
|
|
30
|
+
*/
|
|
31
|
+
// ----- module-scoped state. The session is the single mutation point
|
|
32
|
+
// for the in-memory KV-cache bookkeeping. -----
|
|
33
|
+
/** What the kv-cache file at a given path is known to hold. */
|
|
34
|
+
const cachedPrefixSchema = z.object({
|
|
35
|
+
/** Number of chat messages the file on disk is known to cover. */
|
|
36
|
+
messages: z.number().int().nonnegative(),
|
|
37
|
+
/**
|
|
38
|
+
* Whether a static tool block was rendered into that prefix. Tracked
|
|
39
|
+
* rather than inferred from `messages`, because a committed turn is not
|
|
40
|
+
* proof that its tool block reached the model: the addon drops tools and
|
|
41
|
+
* still returns a usable prompt when the chat template rejects them.
|
|
42
|
+
*/
|
|
43
|
+
toolBlock: z.boolean()
|
|
44
|
+
});
|
|
45
|
+
// The sidecar is written after the addon has already saved the `.bin`, so a
|
|
46
|
+
// crash between the two leaves a boundary that describes an older file. The
|
|
47
|
+
// `.bin` size is a function of the tokens it holds: it is unchanged when the
|
|
48
|
+
// addon re-saves the same state (cancel rewind, key switch) and larger once
|
|
49
|
+
// another turn landed in it, which is the case the fingerprint must catch.
|
|
50
|
+
const persistedPrefixSchema = cachedPrefixSchema.extend({
|
|
51
|
+
binSize: z.number().int().nonnegative()
|
|
52
|
+
});
|
|
13
53
|
/**
|
|
14
54
|
* What the kv-cache file on disk is known to cover, keyed by cache path.
|
|
15
|
-
* Written by `commitTurn`,
|
|
16
|
-
* `
|
|
17
|
-
* `kv-cache-state.ts` still holds: an entry is
|
|
18
|
-
* corresponding `.bin` file is considered trustworthy
|
|
19
|
-
*
|
|
20
|
-
*
|
|
55
|
+
* Written by `commitTurn`, deleted by `rollback`, `delete`,
|
|
56
|
+
* `dropStaleSavedCount` and the orphan clear in `beginTurn`. The same
|
|
57
|
+
* INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
|
|
58
|
+
* present only when the corresponding `.bin` file is considered trustworthy,
|
|
59
|
+
* so any turn that leaves the file holding something the boundary does not
|
|
60
|
+
* describe has to remove it.
|
|
61
|
+
*
|
|
62
|
+
* This map outranks the sidecar. Both are written by the same commit, so an
|
|
63
|
+
* entry here is never the older copy, and `restorePersistedCache` reads the
|
|
64
|
+
* file only to repopulate an empty map after a restart.
|
|
21
65
|
*/
|
|
22
66
|
const cachedPrefixes = new Map();
|
|
67
|
+
const PREFIX_SIDECAR_SUFFIX = '.meta.json';
|
|
68
|
+
function prefixSidecarPath(cachePath) {
|
|
69
|
+
return `${cachePath}${PREFIX_SIDECAR_SUFFIX}`;
|
|
70
|
+
}
|
|
71
|
+
async function writePrefixSidecar(cachePath, prefix, logger) {
|
|
72
|
+
try {
|
|
73
|
+
const { size } = await fsPromises.stat(cachePath);
|
|
74
|
+
await fsPromises.writeFile(prefixSidecarPath(cachePath), JSON.stringify({ ...prefix, binSize: size }));
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
// Only the file goes: a sidecar left from the previous commit describes a
|
|
78
|
+
// shorter `.bin` than the one now on disk. The in-memory boundary is the
|
|
79
|
+
// one this process keeps slicing against and stays.
|
|
80
|
+
await removePrefixSidecar(cachePath);
|
|
81
|
+
logger.warn(`[kv-cache] Failed to persist saved-message boundary; this process stays warm but a restart will start the cache cold. path=${cachePath} error=${error instanceof Error ? error.message : String(error)}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// Anything unreadable, malformed, or describing a `.bin` of a different size
|
|
85
|
+
// is discarded: the cache is then used from a cold boundary, which is the
|
|
86
|
+
// pre-sidecar behaviour. Only ever consulted for a path `cachedPrefixes` has
|
|
87
|
+
// no entry for, so it can never contradict a live boundary. Never throws: the
|
|
88
|
+
// caller holds the cache-path write lock and a `.bin` can vanish under it
|
|
89
|
+
// (`deleteKvCacheState` takes no locks).
|
|
90
|
+
async function readPrefixSidecar(cachePath) {
|
|
91
|
+
let parsed;
|
|
92
|
+
let binSize;
|
|
93
|
+
try {
|
|
94
|
+
const raw = await fsPromises.readFile(prefixSidecarPath(cachePath), 'utf8');
|
|
95
|
+
const result = persistedPrefixSchema.safeParse(JSON.parse(raw));
|
|
96
|
+
if (!result.success)
|
|
97
|
+
return null;
|
|
98
|
+
parsed = result.data;
|
|
99
|
+
binSize = (await fsPromises.stat(cachePath)).size;
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
if (binSize !== parsed.binSize) {
|
|
105
|
+
// The `.bin` changed after this boundary was recorded, so the file is
|
|
106
|
+
// stale whatever wrote it. Only the file: the caller guarantees the map
|
|
107
|
+
// holds nothing for this path.
|
|
108
|
+
await removePrefixSidecar(cachePath);
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
return { messages: parsed.messages, toolBlock: parsed.toolBlock };
|
|
112
|
+
}
|
|
113
|
+
/** Drops the persisted boundary, leaving whatever is in memory alone. */
|
|
114
|
+
async function removePrefixSidecar(cachePath) {
|
|
115
|
+
try {
|
|
116
|
+
await fsPromises.unlink(prefixSidecarPath(cachePath));
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// No sidecar for this path.
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/** Drops the boundary everywhere: the cache it describes is gone or untrusted. */
|
|
123
|
+
async function forgetPrefix(cachePath) {
|
|
124
|
+
cachedPrefixes.delete(cachePath);
|
|
125
|
+
await removePrefixSidecar(cachePath);
|
|
126
|
+
}
|
|
23
127
|
/**
|
|
24
|
-
* In-memory registry of caches
|
|
25
|
-
* defers disk writes, so the absence of a `.bin` file on
|
|
26
|
-
* proof that
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* path in `beginTurn`.
|
|
128
|
+
* In-memory registry of caches established this session, recorded once a save
|
|
129
|
+
* is verified. The addon defers disk writes, so the absence of a `.bin` file on
|
|
130
|
+
* disk isn't proof that no cache exists for this process. Keyed by the resolved
|
|
131
|
+
* cache path, so aliased keys that name one file share an entry and on-disk
|
|
132
|
+
* caches from older process runs still hit the lazy-load path in `beginTurn`.
|
|
30
133
|
*/
|
|
31
134
|
const initializedCaches = new Set();
|
|
32
135
|
const activeCachePaths = new Map();
|
|
@@ -128,6 +231,25 @@ function isCacheKeyActive(cacheKey) {
|
|
|
128
231
|
function getAutoCacheMaxBytes() {
|
|
129
232
|
return isMobile() ? MOBILE_AUTO_CACHE_MAX_BYTES : DESKTOP_AUTO_CACHE_MAX_BYTES;
|
|
130
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* Deletes planned auto-cache keys and reports how many went.
|
|
236
|
+
*
|
|
237
|
+
* Planning and eviction are separate steps, so a turn can take a key in
|
|
238
|
+
* between: each one is re-checked against the active paths under the
|
|
239
|
+
* cache-state lock rather than trusted from the plan.
|
|
240
|
+
*/
|
|
241
|
+
async function evictPlannedAutoCaches(cacheKeys) {
|
|
242
|
+
let evicted = 0;
|
|
243
|
+
await withCacheStateLock(async () => {
|
|
244
|
+
for (const cacheKey of cacheKeys) {
|
|
245
|
+
if (isCacheKeyActive(cacheKey))
|
|
246
|
+
continue;
|
|
247
|
+
await deleteKvCacheState({ kvCacheKey: cacheKey });
|
|
248
|
+
evicted++;
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
return evicted;
|
|
252
|
+
}
|
|
131
253
|
async function withCacheStateLock(operation) {
|
|
132
254
|
const previous = cacheStateLockTail;
|
|
133
255
|
let releaseLock = () => { };
|
|
@@ -156,20 +278,12 @@ async function maybeSweepAutoCaches(logger, overrides) {
|
|
|
156
278
|
const sweep = async () => {
|
|
157
279
|
try {
|
|
158
280
|
const cacheKeys = await planAutoCacheEvictions({
|
|
159
|
-
activeCachePaths:
|
|
281
|
+
activeCachePaths: snapshotActivePaths(),
|
|
160
282
|
maxBytes: overrides?.maxBytes ?? getAutoCacheMaxBytes(),
|
|
161
283
|
maxIdleMs: overrides?.maxIdleMs ?? AUTO_CACHE_MAX_IDLE_MS,
|
|
162
284
|
nowMs
|
|
163
285
|
});
|
|
164
|
-
|
|
165
|
-
await withCacheStateLock(async () => {
|
|
166
|
-
for (const cacheKey of cacheKeys) {
|
|
167
|
-
if (isCacheKeyActive(cacheKey))
|
|
168
|
-
continue;
|
|
169
|
-
await deleteKvCacheState({ kvCacheKey: cacheKey });
|
|
170
|
-
evictionCount++;
|
|
171
|
-
}
|
|
172
|
-
});
|
|
286
|
+
const evictionCount = await evictPlannedAutoCaches(cacheKeys);
|
|
173
287
|
if (evictionCount > 0) {
|
|
174
288
|
logger.debug(`[kv-cache] Evicted ${evictionCount} inactive auto-cache entries`);
|
|
175
289
|
}
|
|
@@ -192,6 +306,30 @@ function scheduleAutoCacheSweep(logger) {
|
|
|
192
306
|
logger.warn(`[kv-cache] Failed to schedule auto-cache retention sweep: ${error instanceof Error ? error.message : String(error)}`);
|
|
193
307
|
});
|
|
194
308
|
}
|
|
309
|
+
/**
|
|
310
|
+
* Drops every auto cache no turn is holding, whatever its age or size.
|
|
311
|
+
*
|
|
312
|
+
* The on-demand counterpart to `maybeSweepAutoCaches`, which applies the
|
|
313
|
+
* standing quota and TTL instead. Reusing the planner keeps the "is this an
|
|
314
|
+
* auto cache" decision in one place: only `.auto-cache-<key>` markers are
|
|
315
|
+
* considered, so caller-owned named caches stay out of scope.
|
|
316
|
+
*/
|
|
317
|
+
async function deleteInactiveAutoCaches() {
|
|
318
|
+
const cacheKeys = await planAutoCacheEvictions({
|
|
319
|
+
activeCachePaths: snapshotActivePaths(),
|
|
320
|
+
// The quota pass stops at `retainedBytes <= maxBytes`, and a total reaches
|
|
321
|
+
// 0 while zero-byte caches remain (an empty `.bin` from an interrupted save).
|
|
322
|
+
// A negative quota is unreachable, so every inactive entry is selected.
|
|
323
|
+
maxBytes: -1,
|
|
324
|
+
// No age threshold: the quota pass above already selects everything.
|
|
325
|
+
maxIdleMs: 0,
|
|
326
|
+
nowMs: Date.now()
|
|
327
|
+
});
|
|
328
|
+
const evicted = await evictPlannedAutoCaches(cacheKeys);
|
|
329
|
+
if (evicted > 0) {
|
|
330
|
+
moduleLogger.debug(`[kv-cache] Reclaimed ${evicted} inactive auto-cache entries`);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
195
333
|
// ----- factory -----
|
|
196
334
|
/**
|
|
197
335
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
@@ -219,7 +357,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
219
357
|
releaseWriteLock,
|
|
220
358
|
committed: false,
|
|
221
359
|
rolledBack: false,
|
|
222
|
-
|
|
360
|
+
createdByThisTurn: false
|
|
223
361
|
});
|
|
224
362
|
markCachePathActive(cachePath);
|
|
225
363
|
return handle;
|
|
@@ -241,7 +379,6 @@ export function createKvCacheSession(modelId, options) {
|
|
|
241
379
|
}
|
|
242
380
|
throw error;
|
|
243
381
|
}
|
|
244
|
-
// A turn cancelled by the time it holds the lock must not prime (native work).
|
|
245
382
|
// getCacheFilePath already mkdir'd the parent, so prune it before surfacing
|
|
246
383
|
// the cancellation the plugin rides.
|
|
247
384
|
if (input.signal?.aborted) {
|
|
@@ -249,36 +386,25 @@ export function createKvCacheSession(modelId, options) {
|
|
|
249
386
|
releaseWriteLock();
|
|
250
387
|
throw new CacheLockAbortError(input.signal.reason);
|
|
251
388
|
}
|
|
389
|
+
// In-memory registry check first — the addon defers disk writes, so a
|
|
390
|
+
// just-saved cache may not yet exist on disk. If the in-memory flag isn't
|
|
391
|
+
// set, fall back to a filesystem probe so caches surviving across process
|
|
392
|
+
// restarts still hit the reuse path. Resolved before the handle is made so
|
|
393
|
+
// it snapshots the restored boundary.
|
|
394
|
+
let exists = initializedCaches.has(cachePath);
|
|
395
|
+
if (!exists)
|
|
396
|
+
exists = await restorePersistedCache(cachePath);
|
|
397
|
+
// A boundary without its `.bin` describes a cache that no longer exists.
|
|
398
|
+
if (!exists)
|
|
399
|
+
await forgetPrefix(cachePath);
|
|
252
400
|
const handle = makeHandle(cachePath, undefined, releaseWriteLock, input.signal);
|
|
253
401
|
try {
|
|
254
|
-
// In-memory registry check first — the addon defers disk writes, so
|
|
255
|
-
// a freshly-primed cache may not yet exist on disk. If the
|
|
256
|
-
// in-memory flag isn't set, fall back to a filesystem probe so
|
|
257
|
-
// caches surviving across process restarts still hit the reuse path.
|
|
258
|
-
let exists = initializedCaches.has(cachePath);
|
|
259
|
-
if (!exists) {
|
|
260
|
-
try {
|
|
261
|
-
await fsPromises.access(cachePath);
|
|
262
|
-
exists = true;
|
|
263
|
-
initializedCaches.add(cachePath);
|
|
264
|
-
}
|
|
265
|
-
catch {
|
|
266
|
-
exists = false;
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
402
|
logCacheStatus(input.customKey, exists);
|
|
270
403
|
if (!exists) {
|
|
271
|
-
// Recreate the parent dir if a same-key peer's rollback pruned it after
|
|
404
|
+
// Recreate the parent dir if a same-key peer's rollback pruned it after
|
|
405
|
+
// our lock wait, so the addon's own save has somewhere to land.
|
|
272
406
|
await fsPromises.mkdir(path.dirname(cachePath), { recursive: true });
|
|
273
|
-
|
|
274
|
-
// since the acquire-time check — re-check right before native priming.
|
|
275
|
-
// The catch below prunes the directory just created.
|
|
276
|
-
if (input.signal?.aborted)
|
|
277
|
-
throw new CacheLockAbortError(input.signal.reason);
|
|
278
|
-
await input.primeIfMissing(cachePath);
|
|
279
|
-
await verifyPrimedFile(cachePath, logger);
|
|
280
|
-
initializedCaches.add(cachePath);
|
|
281
|
-
turnState.get(handle).freshlyPrimed = true;
|
|
407
|
+
turnState.get(handle).createdByThisTurn = true;
|
|
282
408
|
}
|
|
283
409
|
return handle;
|
|
284
410
|
}
|
|
@@ -316,7 +442,6 @@ export function createKvCacheSession(modelId, options) {
|
|
|
316
442
|
}
|
|
317
443
|
throw error;
|
|
318
444
|
}
|
|
319
|
-
// A turn cancelled by the time it holds the lock must not prime (native work).
|
|
320
445
|
// Discovery already mkdir'd the parent and wrote the auto marker; clean both
|
|
321
446
|
// before surfacing the cancellation.
|
|
322
447
|
if (input.signal?.aborted) {
|
|
@@ -354,17 +479,10 @@ export function createKvCacheSession(modelId, options) {
|
|
|
354
479
|
logCacheStatus('auto', cacheExists);
|
|
355
480
|
try {
|
|
356
481
|
if (!cacheExists) {
|
|
357
|
-
// Recreate the parent dir if a same-file peer's rename pruned it
|
|
482
|
+
// Recreate the parent dir if a same-file peer's rename pruned it, so
|
|
483
|
+
// the addon's own save has somewhere to land.
|
|
358
484
|
await fsPromises.mkdir(path.dirname(cachePath), { recursive: true });
|
|
359
|
-
|
|
360
|
-
// the acquire-time check — re-check right before native priming. The
|
|
361
|
-
// catch below prunes the directory and auto marker.
|
|
362
|
-
if (input.signal?.aborted)
|
|
363
|
-
throw new CacheLockAbortError(input.signal.reason);
|
|
364
|
-
await input.primeIfMissing(cachePath);
|
|
365
|
-
await verifyPrimedFile(cachePath, logger);
|
|
366
|
-
initializedCaches.add(cachePath);
|
|
367
|
-
turnState.get(handle).freshlyPrimed = true;
|
|
485
|
+
turnState.get(handle).createdByThisTurn = true;
|
|
368
486
|
}
|
|
369
487
|
return handle;
|
|
370
488
|
}
|
|
@@ -397,13 +515,13 @@ export function createKvCacheSession(modelId, options) {
|
|
|
397
515
|
// at the same path. Verify the file persisted (the addon
|
|
398
516
|
// currently swallows save errors — see TODO in
|
|
399
517
|
// `verifySaveAndRecord`) and record the new boundary.
|
|
400
|
-
const
|
|
518
|
+
const prefix = { messages: result.messageCount, toolBlock: result.toolBlockCached };
|
|
519
|
+
const ok = await verifySaveAndRecord(state.cachePath, prefix);
|
|
401
520
|
if (!ok) {
|
|
402
|
-
// The expected save didn't land — treat the turn as a rollback
|
|
403
|
-
// so the next turn re-primes cleanly.
|
|
404
521
|
await runRollback(state);
|
|
405
522
|
return;
|
|
406
523
|
}
|
|
524
|
+
await writePrefixSidecar(state.cachePath, prefix, logger);
|
|
407
525
|
state.committed = true;
|
|
408
526
|
releaseCachePath(state.cachePath);
|
|
409
527
|
state.releaseWriteLock();
|
|
@@ -446,11 +564,11 @@ export function createKvCacheSession(modelId, options) {
|
|
|
446
564
|
releaseCachePath(sourceCachePath);
|
|
447
565
|
state.cachePath = result.targetCachePath;
|
|
448
566
|
state.autoCacheKey = targetCacheKey;
|
|
567
|
+
await forgetPrefix(sourceCachePath);
|
|
449
568
|
await pruneEmptyCacheDirectories(sourceCachePath, snapshotActivePaths());
|
|
450
569
|
if (sourceCacheKey !== undefined) {
|
|
451
570
|
await removeAutoCacheMarkerIfMissing(sourceCacheKey);
|
|
452
571
|
}
|
|
453
|
-
cachedPrefixes.delete(sourceCachePath);
|
|
454
572
|
// state.cachePath was just reassigned to the target; clear the SOURCE
|
|
455
573
|
// entry, not the freshly-valid target.
|
|
456
574
|
initializedCaches.delete(sourceCachePath);
|
|
@@ -461,7 +579,10 @@ export function createKvCacheSession(modelId, options) {
|
|
|
461
579
|
await runRollback(state);
|
|
462
580
|
return;
|
|
463
581
|
}
|
|
464
|
-
const ok = await verifySaveAndRecord(result.targetCachePath,
|
|
582
|
+
const ok = await verifySaveAndRecord(result.targetCachePath, {
|
|
583
|
+
messages: result.messageCount,
|
|
584
|
+
toolBlock: result.toolBlockCached
|
|
585
|
+
});
|
|
465
586
|
if (!ok) {
|
|
466
587
|
// Rename succeeded but the file isn't where we expected. Roll back via
|
|
467
588
|
// the target path instead of the (now-empty) source.
|
|
@@ -494,9 +615,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
494
615
|
return;
|
|
495
616
|
if (state.committed || state.rolledBack)
|
|
496
617
|
return;
|
|
497
|
-
//
|
|
498
|
-
|
|
499
|
-
if (state.freshlyPrimed) {
|
|
618
|
+
// Nothing committed exists to keep, so a failed first turn is destructive.
|
|
619
|
+
if (state.createdByThisTurn) {
|
|
500
620
|
await runRollback(state);
|
|
501
621
|
return;
|
|
502
622
|
}
|
|
@@ -515,8 +635,13 @@ export function createKvCacheSession(modelId, options) {
|
|
|
515
635
|
await fsPromises.unlink(state.cachePath);
|
|
516
636
|
}
|
|
517
637
|
catch (unlinkError) {
|
|
518
|
-
|
|
638
|
+
// A turn that fails before the addon's save has no file to remove, so
|
|
639
|
+
// ENOENT leaves no stale state behind and is not worth a warning.
|
|
640
|
+
if (unlinkError.code !== 'ENOENT') {
|
|
641
|
+
logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
|
|
642
|
+
}
|
|
519
643
|
}
|
|
644
|
+
await forgetPrefix(state.cachePath);
|
|
520
645
|
// Release before pruning so an empty parent can go; a sibling still holding
|
|
521
646
|
// the path keeps it in the active snapshot and protects the directory.
|
|
522
647
|
releaseCachePath(state.cachePath);
|
|
@@ -525,17 +650,16 @@ export function createKvCacheSession(modelId, options) {
|
|
|
525
650
|
await removeAutoCacheMarkerIfMissing(state.autoCacheKey);
|
|
526
651
|
}
|
|
527
652
|
initializedCaches.delete(state.cachePath);
|
|
528
|
-
cachedPrefixes.delete(state.cachePath);
|
|
529
653
|
state.rolledBack = true;
|
|
530
654
|
state.releaseWriteLock();
|
|
531
655
|
if (state.autoCacheKey !== undefined)
|
|
532
656
|
scheduleAutoCacheSweep(logger);
|
|
533
657
|
}
|
|
534
|
-
function dropStaleSavedCount(turn) {
|
|
658
|
+
async function dropStaleSavedCount(turn) {
|
|
535
659
|
const state = turnState.get(turn);
|
|
536
660
|
if (!state)
|
|
537
661
|
return;
|
|
538
|
-
|
|
662
|
+
await forgetPrefix(state.cachePath);
|
|
539
663
|
}
|
|
540
664
|
return {
|
|
541
665
|
beginTurn,
|
|
@@ -548,9 +672,10 @@ export function createKvCacheSession(modelId, options) {
|
|
|
548
672
|
// ----- module-level administrative API -----
|
|
549
673
|
/**
|
|
550
674
|
* Atomically delete every layer of KV-cache state for a
|
|
551
|
-
* `(kvCacheKey, modelId)` pair,
|
|
552
|
-
*
|
|
553
|
-
* turn-scoped
|
|
675
|
+
* `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
|
|
676
|
+
* caches no turn is holding. Single entry point — the only mutation
|
|
677
|
+
* point for cross-model state outside of turn-scoped
|
|
678
|
+
* `commitTurn`/`rollback`.
|
|
554
679
|
*
|
|
555
680
|
* Why this isn't a method on `KvCacheSession`: deletes are
|
|
556
681
|
* cross-model (`all: true` has no model; the keyed form has
|
|
@@ -575,10 +700,14 @@ export function createKvCacheSession(modelId, options) {
|
|
|
575
700
|
* cleared without a lock, so an interleaving that lands the delete's
|
|
576
701
|
* in-memory cleanup after a concurrent turn has already renamed/committed
|
|
577
702
|
* its file splits state: the file stays on disk while its saved-count and
|
|
578
|
-
* init flag are cleared, so the next turn
|
|
579
|
-
*
|
|
703
|
+
* init flag are cleared, so the next turn loads the file and reports
|
|
704
|
+
* `savedCount=0`. Callers must not delete a key that is in active use.
|
|
705
|
+
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
706
|
+
* and clears each one under the cache-state lock.
|
|
580
707
|
*/
|
|
581
708
|
export async function deleteKvCacheState(target) {
|
|
709
|
+
if ('auto' in target)
|
|
710
|
+
return deleteInactiveAutoCaches();
|
|
582
711
|
if ('all' in target) {
|
|
583
712
|
const removed = await deleteCacheUtil({ all: true });
|
|
584
713
|
cachedPrefixes.clear();
|
|
@@ -609,62 +738,6 @@ export async function deleteKvCacheState(target) {
|
|
|
609
738
|
clearInitializedCachesByPrefix(removedPath, path.sep);
|
|
610
739
|
}
|
|
611
740
|
// ----- private helpers -----
|
|
612
|
-
/**
|
|
613
|
-
* Verify that the addon actually persisted a usable cache file after a
|
|
614
|
-
* prime. Mirrors the `verifySaveAndRecord` access-probe used at commit
|
|
615
|
-
* time, applied at prime time so the session doesn't mark a cache
|
|
616
|
-
* `initializedCaches.add(...)` against a path that's missing or empty
|
|
617
|
-
* on disk.
|
|
618
|
-
*
|
|
619
|
-
* Failure modes this catches:
|
|
620
|
-
*
|
|
621
|
-
* - The addon's `model.run({ saveSessionPath })` was interrupted
|
|
622
|
-
* before the save call ran (e.g. signal abort during prefill); the
|
|
623
|
-
* prime closure resolves cleanly because addon save errors are not
|
|
624
|
-
* propagated, but no file is on disk.
|
|
625
|
-
* - The addon's `llama_state_save_file` was called but produced an
|
|
626
|
-
* empty file (out-of-space / fs error swallowed by the addon).
|
|
627
|
-
*
|
|
628
|
-
* Failure modes this does **NOT** catch:
|
|
629
|
-
*
|
|
630
|
-
* - A partial-but-nonzero file written by the addon (e.g. header +
|
|
631
|
-
* truncated KV state). Catching this requires either an
|
|
632
|
-
* addon-side change (have `CacheManager::writeCacheFile` check the
|
|
633
|
-
* return value of `llama_state_save_file` and throw on failure) or
|
|
634
|
-
* a structural hash check we can't currently compute
|
|
635
|
-
* engine-side. Filed as a follow-up — see `cache-api.md` in the addon
|
|
636
|
-
* repo / tracking ticket.
|
|
637
|
-
*
|
|
638
|
-
* On failure we best-effort `unlink` an empty leftover file (so the
|
|
639
|
-
* next existence probe doesn't trust it) and throw — the handler in
|
|
640
|
-
* `completion-stream.ts` lets the error propagate up and no
|
|
641
|
-
* `initializedCaches` entry is recorded.
|
|
642
|
-
*/
|
|
643
|
-
async function verifyPrimedFile(cachePath, logger) {
|
|
644
|
-
let stats;
|
|
645
|
-
try {
|
|
646
|
-
stats = await fsPromises.stat(cachePath);
|
|
647
|
-
}
|
|
648
|
-
catch (statError) {
|
|
649
|
-
// ENOENT is the common case here — addon prime returned without
|
|
650
|
-
// calling save (most often: signal abort during prefill).
|
|
651
|
-
await pruneEmptyCacheDirectories(cachePath, snapshotActivePaths());
|
|
652
|
-
throw new Error(`[kv-cache] prime closure resolved but no cache file was written. path=${cachePath} cause=${statError instanceof Error ? statError.message : String(statError)}`);
|
|
653
|
-
}
|
|
654
|
-
if (stats.size === 0) {
|
|
655
|
-
// Best-effort cleanup so a future probe doesn't trust the empty
|
|
656
|
-
// file. Unlink failure is non-fatal — we still throw on the
|
|
657
|
-
// primary "prime didn't persist" condition.
|
|
658
|
-
try {
|
|
659
|
-
await fsPromises.unlink(cachePath);
|
|
660
|
-
await pruneEmptyCacheDirectories(cachePath, snapshotActivePaths());
|
|
661
|
-
}
|
|
662
|
-
catch (unlinkError) {
|
|
663
|
-
logger.warn(`[kv-cache] Failed to remove empty primed cache file. path=${cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
|
|
664
|
-
}
|
|
665
|
-
throw new Error(`[kv-cache] prime closure resolved but cache file is empty. path=${cachePath}`);
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
741
|
/**
|
|
669
742
|
* Verify the addon actually persisted the cache file before recording
|
|
670
743
|
* its message count. The addon currently swallows write errors
|
|
@@ -676,10 +749,11 @@ async function verifyPrimedFile(cachePath, logger) {
|
|
|
676
749
|
* false), drop the `access()` probe and wrap the `model.run()` call in
|
|
677
750
|
* a real try/catch that forwards the error.
|
|
678
751
|
*/
|
|
679
|
-
async function verifySaveAndRecord(cachePath,
|
|
752
|
+
async function verifySaveAndRecord(cachePath, prefix) {
|
|
680
753
|
try {
|
|
681
754
|
await fsPromises.access(cachePath);
|
|
682
|
-
cachedPrefixes.set(cachePath,
|
|
755
|
+
cachedPrefixes.set(cachePath, prefix);
|
|
756
|
+
initializedCaches.add(cachePath);
|
|
683
757
|
return true;
|
|
684
758
|
}
|
|
685
759
|
catch (err) {
|
|
@@ -688,6 +762,30 @@ async function verifySaveAndRecord(cachePath, messageCount, toolBlockCached) {
|
|
|
688
762
|
return false;
|
|
689
763
|
}
|
|
690
764
|
}
|
|
765
|
+
/**
|
|
766
|
+
* Adopt a `.bin` left by an earlier process run: record it as established and
|
|
767
|
+
* pick up the boundary committed alongside it. Returns whether the file exists.
|
|
768
|
+
*
|
|
769
|
+
* An auto-rename commits its target into `cachedPrefixes` without an init
|
|
770
|
+
* flag, so this can run for a path the map already knows. The map wins there
|
|
771
|
+
* and the sidecar is left unread, which keeps the two from ever having to be
|
|
772
|
+
* reconciled.
|
|
773
|
+
*/
|
|
774
|
+
async function restorePersistedCache(cachePath) {
|
|
775
|
+
try {
|
|
776
|
+
await fsPromises.access(cachePath);
|
|
777
|
+
}
|
|
778
|
+
catch {
|
|
779
|
+
return false;
|
|
780
|
+
}
|
|
781
|
+
initializedCaches.add(cachePath);
|
|
782
|
+
if (cachedPrefixes.has(cachePath))
|
|
783
|
+
return true;
|
|
784
|
+
const prefix = await readPrefixSidecar(cachePath);
|
|
785
|
+
if (prefix !== null)
|
|
786
|
+
cachedPrefixes.set(cachePath, prefix);
|
|
787
|
+
return true;
|
|
788
|
+
}
|
|
691
789
|
function clearCachedMessageCountsByPrefix(prefix, sep) {
|
|
692
790
|
if (!prefix) {
|
|
693
791
|
cachedPrefixes.clear();
|
|
@@ -733,6 +831,12 @@ export const __kvCacheSessionTestHooks = {
|
|
|
733
831
|
getToolBlockCachedForTest(cachePath) {
|
|
734
832
|
return cachedPrefixes.get(cachePath)?.toolBlock ?? false;
|
|
735
833
|
},
|
|
834
|
+
getPrefixSidecarPathForTest(cachePath) {
|
|
835
|
+
return prefixSidecarPath(cachePath);
|
|
836
|
+
},
|
|
837
|
+
readPrefixSidecarForTest(cachePath) {
|
|
838
|
+
return readPrefixSidecar(cachePath);
|
|
839
|
+
},
|
|
736
840
|
hasInitializedPath(cachePath) {
|
|
737
841
|
return initializedCaches.has(cachePath);
|
|
738
842
|
},
|
|
@@ -48,10 +48,17 @@ export declare function shouldCommitCachedTurn(context: CacheCommitContext): boo
|
|
|
48
48
|
* attachment probing). Kept here so the decision can be unit-tested in
|
|
49
49
|
* isolation.
|
|
50
50
|
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
51
|
+
* A non-zero `savedCount` is itself the proof that a cache holds a rendered
|
|
52
|
+
* prefix: it is recorded only after a save is verified. It means the prefix
|
|
53
|
+
* covers that many messages, so slicing from it drops exactly what the cache
|
|
54
|
+
* supplies — the system message among them. Without a usable boundary the
|
|
55
|
+
* cache supplies nothing, so the whole history goes to the addon, system
|
|
56
|
+
* message included: anything held back would simply be missing from the
|
|
57
|
+
* prompt.
|
|
58
|
+
*
|
|
59
|
+
* The regression guard: a non-zero `savedCount` that slices the history down
|
|
60
|
+
* to an empty array is stale, and the caller resends the full history rather
|
|
54
61
|
* than handing the model an empty payload.
|
|
55
62
|
*/
|
|
56
|
-
export declare function decideCachedHistorySlice(savedCount: number,
|
|
63
|
+
export declare function decideCachedHistorySlice(savedCount: number, history: HistoryMessage[]): HistorySliceDecision;
|
|
57
64
|
export {};
|
|
@@ -30,29 +30,31 @@ export function shouldCommitCachedTurn(context) {
|
|
|
30
30
|
* attachment probing). Kept here so the decision can be unit-tested in
|
|
31
31
|
* isolation.
|
|
32
32
|
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
33
|
+
* A non-zero `savedCount` is itself the proof that a cache holds a rendered
|
|
34
|
+
* prefix: it is recorded only after a save is verified. It means the prefix
|
|
35
|
+
* covers that many messages, so slicing from it drops exactly what the cache
|
|
36
|
+
* supplies — the system message among them. Without a usable boundary the
|
|
37
|
+
* cache supplies nothing, so the whole history goes to the addon, system
|
|
38
|
+
* message included: anything held back would simply be missing from the
|
|
39
|
+
* prompt.
|
|
40
|
+
*
|
|
41
|
+
* The regression guard: a non-zero `savedCount` that slices the history down
|
|
42
|
+
* to an empty array is stale, and the caller resends the full history rather
|
|
36
43
|
* than handing the model an empty payload.
|
|
37
44
|
*/
|
|
38
|
-
export function decideCachedHistorySlice(savedCount,
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
messages: history.filter((msg) => msg.role !== 'system'),
|
|
42
|
-
clearStaleCount: false
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
const canSlice = savedCount > 0 && savedCount <= history.length;
|
|
46
|
-
const sliced = canSlice ? history.slice(savedCount) : null;
|
|
45
|
+
export function decideCachedHistorySlice(savedCount, history) {
|
|
46
|
+
const hasCachedPrefix = savedCount > 0 && history.length > 0;
|
|
47
|
+
const sliced = hasCachedPrefix && savedCount <= history.length ? history.slice(savedCount) : null;
|
|
47
48
|
// A non-null slice that is empty means the saved count is stale: the
|
|
48
49
|
// cached turn boundary is claiming the entire current history is
|
|
49
50
|
// already cached, which happens when a previous turn was cancelled
|
|
50
51
|
// mid-decode and still recorded `history.length + 1`. Treat it as a
|
|
51
|
-
// bad state and resend the full
|
|
52
|
+
// bad state and resend the full history.
|
|
52
53
|
const useSlice = sliced !== null && sliced.length > 0;
|
|
53
|
-
const messages = useSlice ? sliced : history.filter((msg) => msg.role !== 'system');
|
|
54
54
|
return {
|
|
55
|
-
messages,
|
|
56
|
-
|
|
55
|
+
messages: useSlice ? sliced : history,
|
|
56
|
+
// Only a boundary that was actually consulted can be stale. Absent a
|
|
57
|
+
// history to slice, the count is simply unused and left alone.
|
|
58
|
+
clearStaleCount: hasCachedPrefix && !useSlice
|
|
57
59
|
};
|
|
58
60
|
}
|