@qvac/inference 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +43 -11
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +44 -0
- package/dist/schemas/assess-model-fit.js +40 -0
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +749 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2503 -139
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +28 -25
|
@@ -2,18 +2,42 @@ import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
|
|
|
2
2
|
import { type CacheMessage } from '../../../../utils/index.js';
|
|
3
3
|
import type { Logger } from '../../../../logging/types.js';
|
|
4
4
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
/**
|
|
7
|
+
* Coordinates six KV-cache state layers:
|
|
8
|
+
*
|
|
9
|
+
* 1. `cachedPrefixes` — saved message boundaries, and whether a tool
|
|
10
|
+
* block was rendered into them.
|
|
11
|
+
* 2. `initializedCaches` — caches this process has established.
|
|
12
|
+
* 3. On-disk `.bin` files written by the addon.
|
|
13
|
+
* 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
|
|
14
|
+
* boundary outlives the process; see `readPrefixSidecar`.
|
|
15
|
+
* 5. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
16
|
+
* 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
17
|
+
*
|
|
18
|
+
* Every turn must finish through `commitTurn`, `rollback`, or the
|
|
19
|
+
* non-destructive `releaseTurn` so all inference state stays aligned,
|
|
20
|
+
* the active-path ref is released, and marker metadata follows the
|
|
21
|
+
* cache directory lifecycle.
|
|
22
|
+
*/
|
|
23
|
+
/** What the kv-cache file at a given path is known to hold. */
|
|
24
|
+
declare const cachedPrefixSchema: z.ZodObject<{
|
|
25
|
+
messages: z.ZodNumber;
|
|
26
|
+
toolBlock: z.ZodBoolean;
|
|
27
|
+
}, z.core.$strip>;
|
|
28
|
+
type CachedPrefix = z.infer<typeof cachedPrefixSchema>;
|
|
5
29
|
export interface TurnHandle {
|
|
6
30
|
/** Resolved on-disk cache file path the addon will read from / write to. */
|
|
7
31
|
readonly cachePath: string;
|
|
8
32
|
/**
|
|
9
33
|
* Snapshot of the on-disk saved-message count at `beginTurn` time
|
|
10
|
-
* (0
|
|
11
|
-
*
|
|
34
|
+
* (0 on a cold turn). Consumed by `decideCachedHistorySlice` to pick the
|
|
35
|
+
* message tail for the next addon call.
|
|
12
36
|
*/
|
|
13
37
|
readonly savedCount: number;
|
|
14
38
|
/**
|
|
15
39
|
* Whether the cached prefix already holds a rendered static tool block, so
|
|
16
|
-
* this turn can leave it out of its payload. False on a
|
|
40
|
+
* this turn can leave it out of its payload. False on a cold turn and
|
|
17
41
|
* whenever the previous turn couldn't confirm the block reached the model.
|
|
18
42
|
*/
|
|
19
43
|
readonly toolBlockCached: boolean;
|
|
@@ -24,14 +48,6 @@ export interface BeginCustomTurnInput {
|
|
|
24
48
|
customKey: string;
|
|
25
49
|
/** Hash of system prompt + complete tool definitions. */
|
|
26
50
|
configHash: string;
|
|
27
|
-
/**
|
|
28
|
-
* Prime the cache by sending the system prompt to the addon. Tools are not
|
|
29
|
-
* primed — a prefix with no user turn is not a renderable conversation for
|
|
30
|
-
* every template — so they travel with a turn instead. Called when the cache
|
|
31
|
-
* doesn't exist in-memory OR on disk. Kept as an injected closure so this
|
|
32
|
-
* module has no dependency on the model registry / addon.
|
|
33
|
-
*/
|
|
34
|
-
primeIfMissing: (cachePath: string) => Promise<void>;
|
|
35
51
|
/**
|
|
36
52
|
* Request abort signal. When it aborts while this turn is queued behind a
|
|
37
53
|
* same-file peer's write lock, the wait is abandoned so the request's scope
|
|
@@ -45,8 +61,6 @@ export interface BeginAutoTurnInput {
|
|
|
45
61
|
configHash: string;
|
|
46
62
|
/** Conversation history used to compute the pre-response cache key. */
|
|
47
63
|
history: CacheMessage[];
|
|
48
|
-
/** See `BeginCustomTurnInput.primeIfMissing`. */
|
|
49
|
-
primeIfMissing: (cachePath: string) => Promise<void>;
|
|
50
64
|
/** See `BeginCustomTurnInput.signal`. */
|
|
51
65
|
signal?: AbortSignal;
|
|
52
66
|
}
|
|
@@ -80,10 +94,10 @@ export type CommitResult = StaticCommitResult | AutoRenameCommitResult;
|
|
|
80
94
|
export interface KvCacheSession {
|
|
81
95
|
/**
|
|
82
96
|
* Open a new turn against the cache. Resolves the cache file path,
|
|
83
|
-
*
|
|
84
|
-
* `
|
|
85
|
-
* `
|
|
86
|
-
*
|
|
97
|
+
* ensures its parent directory exists, and returns a `TurnHandle` the
|
|
98
|
+
* handler attaches to `ctx.scope.defer(...)` for the rollback hook. The
|
|
99
|
+
* addon's own `saveCacheToDisk` is what writes the file, so a cold turn
|
|
100
|
+
* opens without one. Auto-cache path resolution is serialized with
|
|
87
101
|
* retention deletion before the handle is returned.
|
|
88
102
|
*/
|
|
89
103
|
beginTurn(input: BeginTurnInput): Promise<TurnHandle>;
|
|
@@ -109,17 +123,17 @@ export interface KvCacheSession {
|
|
|
109
123
|
/**
|
|
110
124
|
* Non-destructive counterpart of `rollback`: releases locks and refs but
|
|
111
125
|
* keeps the committed disk cache and its recorded prefix valid for a retry.
|
|
112
|
-
* A cache
|
|
126
|
+
* A cache this same turn created rolls back instead.
|
|
113
127
|
*/
|
|
114
128
|
releaseTurn(turn: TurnHandle): Promise<void>;
|
|
115
129
|
/**
|
|
116
|
-
* Forget the
|
|
117
|
-
* without unlinking the file or clearing the init flag. Used
|
|
118
|
-
* `decideCachedHistorySlice` detects a stale boundary
|
|
130
|
+
* Forget the saved-message count for the turn's path, in memory and in the
|
|
131
|
+
* sidecar, without unlinking the cache file or clearing the init flag. Used
|
|
132
|
+
* when `decideCachedHistorySlice` detects a stale boundary
|
|
119
133
|
* (`clearStaleCount: true`) — the next turn re-sends the full history
|
|
120
134
|
* but the cache itself is still usable.
|
|
121
135
|
*/
|
|
122
|
-
dropStaleSavedCount(turn: TurnHandle): void
|
|
136
|
+
dropStaleSavedCount(turn: TurnHandle): Promise<void>;
|
|
123
137
|
}
|
|
124
138
|
/**
|
|
125
139
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
@@ -160,8 +174,8 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
160
174
|
* cleared without a lock, so an interleaving that lands the delete's
|
|
161
175
|
* in-memory cleanup after a concurrent turn has already renamed/committed
|
|
162
176
|
* its file splits state: the file stays on disk while its saved-count and
|
|
163
|
-
* init flag are cleared, so the next turn
|
|
164
|
-
*
|
|
177
|
+
* init flag are cleared, so the next turn loads the file and reports
|
|
178
|
+
* `savedCount=0`. Callers must not delete a key that is in active use.
|
|
165
179
|
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
166
180
|
* and clears each one under the cache-state lock.
|
|
167
181
|
*/
|
|
@@ -186,6 +200,8 @@ export declare const __kvCacheSessionTestHooks: {
|
|
|
186
200
|
getSavedCount(cachePath: string): number | undefined;
|
|
187
201
|
setSavedCountForTest(cachePath: string, count: number): void;
|
|
188
202
|
getToolBlockCachedForTest(cachePath: string): boolean;
|
|
203
|
+
getPrefixSidecarPathForTest(cachePath: string): string;
|
|
204
|
+
readPrefixSidecarForTest(cachePath: string): Promise<CachedPrefix | null>;
|
|
189
205
|
hasInitializedPath(cachePath: string): boolean;
|
|
190
206
|
getActivePathCountForTest(cachePath: string): number;
|
|
191
207
|
getLastAutoCacheSweepMsForTest(): number;
|
|
@@ -6,27 +6,130 @@ import { isMobile } from '../../../../runtime/state.js';
|
|
|
6
6
|
import { getKVCacheDir } from '../../../../utils/index.js';
|
|
7
7
|
import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
|
|
8
8
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
9
|
+
import { z } from 'zod';
|
|
9
10
|
// Used by cross-model paths that have no `RequestContext` (e.g.
|
|
10
11
|
// `deleteKvCacheState`). Per-session call sites receive a logger from
|
|
11
12
|
// the caller — typically `withRequestContext(...)`.
|
|
12
13
|
const moduleLogger = getEngineLogger();
|
|
14
|
+
/**
|
|
15
|
+
* Coordinates six KV-cache state layers:
|
|
16
|
+
*
|
|
17
|
+
* 1. `cachedPrefixes` — saved message boundaries, and whether a tool
|
|
18
|
+
* block was rendered into them.
|
|
19
|
+
* 2. `initializedCaches` — caches this process has established.
|
|
20
|
+
* 3. On-disk `.bin` files written by the addon.
|
|
21
|
+
* 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
|
|
22
|
+
* boundary outlives the process; see `readPrefixSidecar`.
|
|
23
|
+
* 5. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
24
|
+
* 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
25
|
+
*
|
|
26
|
+
* Every turn must finish through `commitTurn`, `rollback`, or the
|
|
27
|
+
* non-destructive `releaseTurn` so all inference state stays aligned,
|
|
28
|
+
* the active-path ref is released, and marker metadata follows the
|
|
29
|
+
* cache directory lifecycle.
|
|
30
|
+
*/
|
|
31
|
+
// ----- module-scoped state. The session is the single mutation point
|
|
32
|
+
// for the in-memory KV-cache bookkeeping. -----
|
|
33
|
+
/** What the kv-cache file at a given path is known to hold. */
|
|
34
|
+
const cachedPrefixSchema = z.object({
|
|
35
|
+
/** Number of chat messages the file on disk is known to cover. */
|
|
36
|
+
messages: z.number().int().nonnegative(),
|
|
37
|
+
/**
|
|
38
|
+
* Whether a static tool block was rendered into that prefix. Tracked
|
|
39
|
+
* rather than inferred from `messages`, because a committed turn is not
|
|
40
|
+
* proof that its tool block reached the model: the addon drops tools and
|
|
41
|
+
* still returns a usable prompt when the chat template rejects them.
|
|
42
|
+
*/
|
|
43
|
+
toolBlock: z.boolean()
|
|
44
|
+
});
|
|
45
|
+
// The sidecar is written after the addon has already saved the `.bin`, so a
|
|
46
|
+
// crash between the two leaves a boundary that describes an older file. The
|
|
47
|
+
// `.bin` size is a function of the tokens it holds: it is unchanged when the
|
|
48
|
+
// addon re-saves the same state (cancel rewind, key switch) and larger once
|
|
49
|
+
// another turn landed in it, which is the case the fingerprint must catch.
|
|
50
|
+
const persistedPrefixSchema = cachedPrefixSchema.extend({
|
|
51
|
+
binSize: z.number().int().nonnegative()
|
|
52
|
+
});
|
|
13
53
|
/**
|
|
14
54
|
* What the kv-cache file on disk is known to cover, keyed by cache path.
|
|
15
|
-
* Written by `commitTurn`,
|
|
16
|
-
* `
|
|
17
|
-
* `kv-cache-state.ts` still holds: an entry is
|
|
18
|
-
* corresponding `.bin` file is considered trustworthy
|
|
19
|
-
*
|
|
20
|
-
*
|
|
55
|
+
* Written by `commitTurn`, deleted by `rollback`, `delete`,
|
|
56
|
+
* `dropStaleSavedCount` and the orphan clear in `beginTurn`. The same
|
|
57
|
+
* INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
|
|
58
|
+
* present only when the corresponding `.bin` file is considered trustworthy,
|
|
59
|
+
* so any turn that leaves the file holding something the boundary does not
|
|
60
|
+
* describe has to remove it.
|
|
61
|
+
*
|
|
62
|
+
* This map outranks the sidecar. Both are written by the same commit, so an
|
|
63
|
+
* entry here is never the older copy, and `restorePersistedCache` reads the
|
|
64
|
+
* file only to repopulate an empty map after a restart.
|
|
21
65
|
*/
|
|
22
66
|
const cachedPrefixes = new Map();
|
|
67
|
+
const PREFIX_SIDECAR_SUFFIX = '.meta.json';
|
|
68
|
+
function prefixSidecarPath(cachePath) {
|
|
69
|
+
return `${cachePath}${PREFIX_SIDECAR_SUFFIX}`;
|
|
70
|
+
}
|
|
71
|
+
async function writePrefixSidecar(cachePath, prefix, logger) {
|
|
72
|
+
try {
|
|
73
|
+
const { size } = await fsPromises.stat(cachePath);
|
|
74
|
+
await fsPromises.writeFile(prefixSidecarPath(cachePath), JSON.stringify({ ...prefix, binSize: size }));
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
// Only the file goes: a sidecar left from the previous commit describes a
|
|
78
|
+
// shorter `.bin` than the one now on disk. The in-memory boundary is the
|
|
79
|
+
// one this process keeps slicing against and stays.
|
|
80
|
+
await removePrefixSidecar(cachePath);
|
|
81
|
+
logger.warn(`[kv-cache] Failed to persist saved-message boundary; this process stays warm but a restart will start the cache cold. path=${cachePath} error=${error instanceof Error ? error.message : String(error)}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// Anything unreadable, malformed, or describing a `.bin` of a different size
|
|
85
|
+
// is discarded: the cache is then used from a cold boundary, which is the
|
|
86
|
+
// pre-sidecar behaviour. Only ever consulted for a path `cachedPrefixes` has
|
|
87
|
+
// no entry for, so it can never contradict a live boundary. Never throws: the
|
|
88
|
+
// caller holds the cache-path write lock and a `.bin` can vanish under it
|
|
89
|
+
// (`deleteKvCacheState` takes no locks).
|
|
90
|
+
async function readPrefixSidecar(cachePath) {
|
|
91
|
+
let parsed;
|
|
92
|
+
let binSize;
|
|
93
|
+
try {
|
|
94
|
+
const raw = await fsPromises.readFile(prefixSidecarPath(cachePath), 'utf8');
|
|
95
|
+
const result = persistedPrefixSchema.safeParse(JSON.parse(raw));
|
|
96
|
+
if (!result.success)
|
|
97
|
+
return null;
|
|
98
|
+
parsed = result.data;
|
|
99
|
+
binSize = (await fsPromises.stat(cachePath)).size;
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
if (binSize !== parsed.binSize) {
|
|
105
|
+
// The `.bin` changed after this boundary was recorded, so the file is
|
|
106
|
+
// stale whatever wrote it. Only the file: the caller guarantees the map
|
|
107
|
+
// holds nothing for this path.
|
|
108
|
+
await removePrefixSidecar(cachePath);
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
return { messages: parsed.messages, toolBlock: parsed.toolBlock };
|
|
112
|
+
}
|
|
113
|
+
/** Drops the persisted boundary, leaving whatever is in memory alone. */
|
|
114
|
+
async function removePrefixSidecar(cachePath) {
|
|
115
|
+
try {
|
|
116
|
+
await fsPromises.unlink(prefixSidecarPath(cachePath));
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// No sidecar for this path.
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/** Drops the boundary everywhere: the cache it describes is gone or untrusted. */
|
|
123
|
+
async function forgetPrefix(cachePath) {
|
|
124
|
+
cachedPrefixes.delete(cachePath);
|
|
125
|
+
await removePrefixSidecar(cachePath);
|
|
126
|
+
}
|
|
23
127
|
/**
|
|
24
|
-
* In-memory registry of caches
|
|
25
|
-
* defers disk writes, so the absence of a `.bin` file on
|
|
26
|
-
* proof that
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* path in `beginTurn`.
|
|
128
|
+
* In-memory registry of caches established this session, recorded once a save
|
|
129
|
+
* is verified. The addon defers disk writes, so the absence of a `.bin` file on
|
|
130
|
+
* disk isn't proof that no cache exists for this process. Keyed by the resolved
|
|
131
|
+
* cache path, so aliased keys that name one file share an entry and on-disk
|
|
132
|
+
* caches from older process runs still hit the lazy-load path in `beginTurn`.
|
|
30
133
|
*/
|
|
31
134
|
const initializedCaches = new Set();
|
|
32
135
|
const activeCachePaths = new Map();
|
|
@@ -215,7 +318,7 @@ async function deleteInactiveAutoCaches() {
|
|
|
215
318
|
const cacheKeys = await planAutoCacheEvictions({
|
|
216
319
|
activeCachePaths: snapshotActivePaths(),
|
|
217
320
|
// The quota pass stops at `retainedBytes <= maxBytes`, and a total reaches
|
|
218
|
-
// 0 while zero-byte caches remain (an empty `.bin` from
|
|
321
|
+
// 0 while zero-byte caches remain (an empty `.bin` from an interrupted save).
|
|
219
322
|
// A negative quota is unreachable, so every inactive entry is selected.
|
|
220
323
|
maxBytes: -1,
|
|
221
324
|
// No age threshold: the quota pass above already selects everything.
|
|
@@ -254,7 +357,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
254
357
|
releaseWriteLock,
|
|
255
358
|
committed: false,
|
|
256
359
|
rolledBack: false,
|
|
257
|
-
|
|
360
|
+
createdByThisTurn: false
|
|
258
361
|
});
|
|
259
362
|
markCachePathActive(cachePath);
|
|
260
363
|
return handle;
|
|
@@ -276,7 +379,6 @@ export function createKvCacheSession(modelId, options) {
|
|
|
276
379
|
}
|
|
277
380
|
throw error;
|
|
278
381
|
}
|
|
279
|
-
// A turn cancelled by the time it holds the lock must not prime (native work).
|
|
280
382
|
// getCacheFilePath already mkdir'd the parent, so prune it before surfacing
|
|
281
383
|
// the cancellation the plugin rides.
|
|
282
384
|
if (input.signal?.aborted) {
|
|
@@ -284,36 +386,25 @@ export function createKvCacheSession(modelId, options) {
|
|
|
284
386
|
releaseWriteLock();
|
|
285
387
|
throw new CacheLockAbortError(input.signal.reason);
|
|
286
388
|
}
|
|
389
|
+
// In-memory registry check first — the addon defers disk writes, so a
|
|
390
|
+
// just-saved cache may not yet exist on disk. If the in-memory flag isn't
|
|
391
|
+
// set, fall back to a filesystem probe so caches surviving across process
|
|
392
|
+
// restarts still hit the reuse path. Resolved before the handle is made so
|
|
393
|
+
// it snapshots the restored boundary.
|
|
394
|
+
let exists = initializedCaches.has(cachePath);
|
|
395
|
+
if (!exists)
|
|
396
|
+
exists = await restorePersistedCache(cachePath);
|
|
397
|
+
// A boundary without its `.bin` describes a cache that no longer exists.
|
|
398
|
+
if (!exists)
|
|
399
|
+
await forgetPrefix(cachePath);
|
|
287
400
|
const handle = makeHandle(cachePath, undefined, releaseWriteLock, input.signal);
|
|
288
401
|
try {
|
|
289
|
-
// In-memory registry check first — the addon defers disk writes, so
|
|
290
|
-
// a freshly-primed cache may not yet exist on disk. If the
|
|
291
|
-
// in-memory flag isn't set, fall back to a filesystem probe so
|
|
292
|
-
// caches surviving across process restarts still hit the reuse path.
|
|
293
|
-
let exists = initializedCaches.has(cachePath);
|
|
294
|
-
if (!exists) {
|
|
295
|
-
try {
|
|
296
|
-
await fsPromises.access(cachePath);
|
|
297
|
-
exists = true;
|
|
298
|
-
initializedCaches.add(cachePath);
|
|
299
|
-
}
|
|
300
|
-
catch {
|
|
301
|
-
exists = false;
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
402
|
logCacheStatus(input.customKey, exists);
|
|
305
403
|
if (!exists) {
|
|
306
|
-
// Recreate the parent dir if a same-key peer's rollback pruned it after
|
|
404
|
+
// Recreate the parent dir if a same-key peer's rollback pruned it after
|
|
405
|
+
// our lock wait, so the addon's own save has somewhere to land.
|
|
307
406
|
await fsPromises.mkdir(path.dirname(cachePath), { recursive: true });
|
|
308
|
-
|
|
309
|
-
// since the acquire-time check — re-check right before native priming.
|
|
310
|
-
// The catch below prunes the directory just created.
|
|
311
|
-
if (input.signal?.aborted)
|
|
312
|
-
throw new CacheLockAbortError(input.signal.reason);
|
|
313
|
-
await input.primeIfMissing(cachePath);
|
|
314
|
-
await verifyPrimedFile(cachePath, logger);
|
|
315
|
-
initializedCaches.add(cachePath);
|
|
316
|
-
turnState.get(handle).freshlyPrimed = true;
|
|
407
|
+
turnState.get(handle).createdByThisTurn = true;
|
|
317
408
|
}
|
|
318
409
|
return handle;
|
|
319
410
|
}
|
|
@@ -351,7 +442,6 @@ export function createKvCacheSession(modelId, options) {
|
|
|
351
442
|
}
|
|
352
443
|
throw error;
|
|
353
444
|
}
|
|
354
|
-
// A turn cancelled by the time it holds the lock must not prime (native work).
|
|
355
445
|
// Discovery already mkdir'd the parent and wrote the auto marker; clean both
|
|
356
446
|
// before surfacing the cancellation.
|
|
357
447
|
if (input.signal?.aborted) {
|
|
@@ -389,17 +479,10 @@ export function createKvCacheSession(modelId, options) {
|
|
|
389
479
|
logCacheStatus('auto', cacheExists);
|
|
390
480
|
try {
|
|
391
481
|
if (!cacheExists) {
|
|
392
|
-
// Recreate the parent dir if a same-file peer's rename pruned it
|
|
482
|
+
// Recreate the parent dir if a same-file peer's rename pruned it, so
|
|
483
|
+
// the addon's own save has somewhere to land.
|
|
393
484
|
await fsPromises.mkdir(path.dirname(cachePath), { recursive: true });
|
|
394
|
-
|
|
395
|
-
// the acquire-time check — re-check right before native priming. The
|
|
396
|
-
// catch below prunes the directory and auto marker.
|
|
397
|
-
if (input.signal?.aborted)
|
|
398
|
-
throw new CacheLockAbortError(input.signal.reason);
|
|
399
|
-
await input.primeIfMissing(cachePath);
|
|
400
|
-
await verifyPrimedFile(cachePath, logger);
|
|
401
|
-
initializedCaches.add(cachePath);
|
|
402
|
-
turnState.get(handle).freshlyPrimed = true;
|
|
485
|
+
turnState.get(handle).createdByThisTurn = true;
|
|
403
486
|
}
|
|
404
487
|
return handle;
|
|
405
488
|
}
|
|
@@ -432,13 +515,13 @@ export function createKvCacheSession(modelId, options) {
|
|
|
432
515
|
// at the same path. Verify the file persisted (the addon
|
|
433
516
|
// currently swallows save errors — see TODO in
|
|
434
517
|
// `verifySaveAndRecord`) and record the new boundary.
|
|
435
|
-
const
|
|
518
|
+
const prefix = { messages: result.messageCount, toolBlock: result.toolBlockCached };
|
|
519
|
+
const ok = await verifySaveAndRecord(state.cachePath, prefix);
|
|
436
520
|
if (!ok) {
|
|
437
|
-
// The expected save didn't land — treat the turn as a rollback
|
|
438
|
-
// so the next turn re-primes cleanly.
|
|
439
521
|
await runRollback(state);
|
|
440
522
|
return;
|
|
441
523
|
}
|
|
524
|
+
await writePrefixSidecar(state.cachePath, prefix, logger);
|
|
442
525
|
state.committed = true;
|
|
443
526
|
releaseCachePath(state.cachePath);
|
|
444
527
|
state.releaseWriteLock();
|
|
@@ -481,11 +564,11 @@ export function createKvCacheSession(modelId, options) {
|
|
|
481
564
|
releaseCachePath(sourceCachePath);
|
|
482
565
|
state.cachePath = result.targetCachePath;
|
|
483
566
|
state.autoCacheKey = targetCacheKey;
|
|
567
|
+
await forgetPrefix(sourceCachePath);
|
|
484
568
|
await pruneEmptyCacheDirectories(sourceCachePath, snapshotActivePaths());
|
|
485
569
|
if (sourceCacheKey !== undefined) {
|
|
486
570
|
await removeAutoCacheMarkerIfMissing(sourceCacheKey);
|
|
487
571
|
}
|
|
488
|
-
cachedPrefixes.delete(sourceCachePath);
|
|
489
572
|
// state.cachePath was just reassigned to the target; clear the SOURCE
|
|
490
573
|
// entry, not the freshly-valid target.
|
|
491
574
|
initializedCaches.delete(sourceCachePath);
|
|
@@ -496,7 +579,10 @@ export function createKvCacheSession(modelId, options) {
|
|
|
496
579
|
await runRollback(state);
|
|
497
580
|
return;
|
|
498
581
|
}
|
|
499
|
-
const ok = await verifySaveAndRecord(result.targetCachePath,
|
|
582
|
+
const ok = await verifySaveAndRecord(result.targetCachePath, {
|
|
583
|
+
messages: result.messageCount,
|
|
584
|
+
toolBlock: result.toolBlockCached
|
|
585
|
+
});
|
|
500
586
|
if (!ok) {
|
|
501
587
|
// Rename succeeded but the file isn't where we expected. Roll back via
|
|
502
588
|
// the target path instead of the (now-empty) source.
|
|
@@ -529,9 +615,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
529
615
|
return;
|
|
530
616
|
if (state.committed || state.rolledBack)
|
|
531
617
|
return;
|
|
532
|
-
//
|
|
533
|
-
|
|
534
|
-
if (state.freshlyPrimed) {
|
|
618
|
+
// Nothing committed exists to keep, so a failed first turn is destructive.
|
|
619
|
+
if (state.createdByThisTurn) {
|
|
535
620
|
await runRollback(state);
|
|
536
621
|
return;
|
|
537
622
|
}
|
|
@@ -550,8 +635,13 @@ export function createKvCacheSession(modelId, options) {
|
|
|
550
635
|
await fsPromises.unlink(state.cachePath);
|
|
551
636
|
}
|
|
552
637
|
catch (unlinkError) {
|
|
553
|
-
|
|
638
|
+
// A turn that fails before the addon's save has no file to remove, so
|
|
639
|
+
// ENOENT leaves no stale state behind and is not worth a warning.
|
|
640
|
+
if (unlinkError.code !== 'ENOENT') {
|
|
641
|
+
logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
|
|
642
|
+
}
|
|
554
643
|
}
|
|
644
|
+
await forgetPrefix(state.cachePath);
|
|
555
645
|
// Release before pruning so an empty parent can go; a sibling still holding
|
|
556
646
|
// the path keeps it in the active snapshot and protects the directory.
|
|
557
647
|
releaseCachePath(state.cachePath);
|
|
@@ -560,17 +650,16 @@ export function createKvCacheSession(modelId, options) {
|
|
|
560
650
|
await removeAutoCacheMarkerIfMissing(state.autoCacheKey);
|
|
561
651
|
}
|
|
562
652
|
initializedCaches.delete(state.cachePath);
|
|
563
|
-
cachedPrefixes.delete(state.cachePath);
|
|
564
653
|
state.rolledBack = true;
|
|
565
654
|
state.releaseWriteLock();
|
|
566
655
|
if (state.autoCacheKey !== undefined)
|
|
567
656
|
scheduleAutoCacheSweep(logger);
|
|
568
657
|
}
|
|
569
|
-
function dropStaleSavedCount(turn) {
|
|
658
|
+
async function dropStaleSavedCount(turn) {
|
|
570
659
|
const state = turnState.get(turn);
|
|
571
660
|
if (!state)
|
|
572
661
|
return;
|
|
573
|
-
|
|
662
|
+
await forgetPrefix(state.cachePath);
|
|
574
663
|
}
|
|
575
664
|
return {
|
|
576
665
|
beginTurn,
|
|
@@ -611,8 +700,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
611
700
|
* cleared without a lock, so an interleaving that lands the delete's
|
|
612
701
|
* in-memory cleanup after a concurrent turn has already renamed/committed
|
|
613
702
|
* its file splits state: the file stays on disk while its saved-count and
|
|
614
|
-
* init flag are cleared, so the next turn
|
|
615
|
-
*
|
|
703
|
+
* init flag are cleared, so the next turn loads the file and reports
|
|
704
|
+
* `savedCount=0`. Callers must not delete a key that is in active use.
|
|
616
705
|
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
617
706
|
* and clears each one under the cache-state lock.
|
|
618
707
|
*/
|
|
@@ -649,62 +738,6 @@ export async function deleteKvCacheState(target) {
|
|
|
649
738
|
clearInitializedCachesByPrefix(removedPath, path.sep);
|
|
650
739
|
}
|
|
651
740
|
// ----- private helpers -----
|
|
652
|
-
/**
|
|
653
|
-
* Verify that the addon actually persisted a usable cache file after a
|
|
654
|
-
* prime. Mirrors the `verifySaveAndRecord` access-probe used at commit
|
|
655
|
-
* time, applied at prime time so the session doesn't mark a cache
|
|
656
|
-
* `initializedCaches.add(...)` against a path that's missing or empty
|
|
657
|
-
* on disk.
|
|
658
|
-
*
|
|
659
|
-
* Failure modes this catches:
|
|
660
|
-
*
|
|
661
|
-
* - The addon's `model.run({ saveSessionPath })` was interrupted
|
|
662
|
-
* before the save call ran (e.g. signal abort during prefill); the
|
|
663
|
-
* prime closure resolves cleanly because addon save errors are not
|
|
664
|
-
* propagated, but no file is on disk.
|
|
665
|
-
* - The addon's `llama_state_save_file` was called but produced an
|
|
666
|
-
* empty file (out-of-space / fs error swallowed by the addon).
|
|
667
|
-
*
|
|
668
|
-
* Failure modes this does **NOT** catch:
|
|
669
|
-
*
|
|
670
|
-
* - A partial-but-nonzero file written by the addon (e.g. header +
|
|
671
|
-
* truncated KV state). Catching this requires either an
|
|
672
|
-
* addon-side change (have `CacheManager::writeCacheFile` check the
|
|
673
|
-
* return value of `llama_state_save_file` and throw on failure) or
|
|
674
|
-
* a structural hash check we can't currently compute
|
|
675
|
-
* engine-side. Filed as a follow-up — see `cache-api.md` in the addon
|
|
676
|
-
* repo / tracking ticket.
|
|
677
|
-
*
|
|
678
|
-
* On failure we best-effort `unlink` an empty leftover file (so the
|
|
679
|
-
* next existence probe doesn't trust it) and throw — the handler in
|
|
680
|
-
* `completion-stream.ts` lets the error propagate up and no
|
|
681
|
-
* `initializedCaches` entry is recorded.
|
|
682
|
-
*/
|
|
683
|
-
async function verifyPrimedFile(cachePath, logger) {
|
|
684
|
-
let stats;
|
|
685
|
-
try {
|
|
686
|
-
stats = await fsPromises.stat(cachePath);
|
|
687
|
-
}
|
|
688
|
-
catch (statError) {
|
|
689
|
-
// ENOENT is the common case here — addon prime returned without
|
|
690
|
-
// calling save (most often: signal abort during prefill).
|
|
691
|
-
await pruneEmptyCacheDirectories(cachePath, snapshotActivePaths());
|
|
692
|
-
throw new Error(`[kv-cache] prime closure resolved but no cache file was written. path=${cachePath} cause=${statError instanceof Error ? statError.message : String(statError)}`);
|
|
693
|
-
}
|
|
694
|
-
if (stats.size === 0) {
|
|
695
|
-
// Best-effort cleanup so a future probe doesn't trust the empty
|
|
696
|
-
// file. Unlink failure is non-fatal — we still throw on the
|
|
697
|
-
// primary "prime didn't persist" condition.
|
|
698
|
-
try {
|
|
699
|
-
await fsPromises.unlink(cachePath);
|
|
700
|
-
await pruneEmptyCacheDirectories(cachePath, snapshotActivePaths());
|
|
701
|
-
}
|
|
702
|
-
catch (unlinkError) {
|
|
703
|
-
logger.warn(`[kv-cache] Failed to remove empty primed cache file. path=${cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
|
|
704
|
-
}
|
|
705
|
-
throw new Error(`[kv-cache] prime closure resolved but cache file is empty. path=${cachePath}`);
|
|
706
|
-
}
|
|
707
|
-
}
|
|
708
741
|
/**
|
|
709
742
|
* Verify the addon actually persisted the cache file before recording
|
|
710
743
|
* its message count. The addon currently swallows write errors
|
|
@@ -716,10 +749,11 @@ async function verifyPrimedFile(cachePath, logger) {
|
|
|
716
749
|
* false), drop the `access()` probe and wrap the `model.run()` call in
|
|
717
750
|
* a real try/catch that forwards the error.
|
|
718
751
|
*/
|
|
719
|
-
async function verifySaveAndRecord(cachePath,
|
|
752
|
+
async function verifySaveAndRecord(cachePath, prefix) {
|
|
720
753
|
try {
|
|
721
754
|
await fsPromises.access(cachePath);
|
|
722
|
-
cachedPrefixes.set(cachePath,
|
|
755
|
+
cachedPrefixes.set(cachePath, prefix);
|
|
756
|
+
initializedCaches.add(cachePath);
|
|
723
757
|
return true;
|
|
724
758
|
}
|
|
725
759
|
catch (err) {
|
|
@@ -728,6 +762,30 @@ async function verifySaveAndRecord(cachePath, messageCount, toolBlockCached) {
|
|
|
728
762
|
return false;
|
|
729
763
|
}
|
|
730
764
|
}
|
|
765
|
+
/**
|
|
766
|
+
* Adopt a `.bin` left by an earlier process run: record it as established and
|
|
767
|
+
* pick up the boundary committed alongside it. Returns whether the file exists.
|
|
768
|
+
*
|
|
769
|
+
* An auto-rename commits its target into `cachedPrefixes` without an init
|
|
770
|
+
* flag, so this can run for a path the map already knows. The map wins there
|
|
771
|
+
* and the sidecar is left unread, which keeps the two from ever having to be
|
|
772
|
+
* reconciled.
|
|
773
|
+
*/
|
|
774
|
+
async function restorePersistedCache(cachePath) {
|
|
775
|
+
try {
|
|
776
|
+
await fsPromises.access(cachePath);
|
|
777
|
+
}
|
|
778
|
+
catch {
|
|
779
|
+
return false;
|
|
780
|
+
}
|
|
781
|
+
initializedCaches.add(cachePath);
|
|
782
|
+
if (cachedPrefixes.has(cachePath))
|
|
783
|
+
return true;
|
|
784
|
+
const prefix = await readPrefixSidecar(cachePath);
|
|
785
|
+
if (prefix !== null)
|
|
786
|
+
cachedPrefixes.set(cachePath, prefix);
|
|
787
|
+
return true;
|
|
788
|
+
}
|
|
731
789
|
function clearCachedMessageCountsByPrefix(prefix, sep) {
|
|
732
790
|
if (!prefix) {
|
|
733
791
|
cachedPrefixes.clear();
|
|
@@ -773,6 +831,12 @@ export const __kvCacheSessionTestHooks = {
|
|
|
773
831
|
getToolBlockCachedForTest(cachePath) {
|
|
774
832
|
return cachedPrefixes.get(cachePath)?.toolBlock ?? false;
|
|
775
833
|
},
|
|
834
|
+
getPrefixSidecarPathForTest(cachePath) {
|
|
835
|
+
return prefixSidecarPath(cachePath);
|
|
836
|
+
},
|
|
837
|
+
readPrefixSidecarForTest(cachePath) {
|
|
838
|
+
return readPrefixSidecar(cachePath);
|
|
839
|
+
},
|
|
776
840
|
hasInitializedPath(cachePath) {
|
|
777
841
|
return initializedCaches.has(cachePath);
|
|
778
842
|
},
|
|
@@ -48,10 +48,17 @@ export declare function shouldCommitCachedTurn(context: CacheCommitContext): boo
|
|
|
48
48
|
* attachment probing). Kept here so the decision can be unit-tested in
|
|
49
49
|
* isolation.
|
|
50
50
|
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
51
|
+
* A non-zero `savedCount` is itself the proof that a cache holds a rendered
|
|
52
|
+
* prefix: it is recorded only after a save is verified. It means the prefix
|
|
53
|
+
* covers that many messages, so slicing from it drops exactly what the cache
|
|
54
|
+
* supplies — the system message among them. Without a usable boundary the
|
|
55
|
+
* cache supplies nothing, so the whole history goes to the addon, system
|
|
56
|
+
* message included: anything held back would simply be missing from the
|
|
57
|
+
* prompt.
|
|
58
|
+
*
|
|
59
|
+
* The regression guard: a non-zero `savedCount` that slices the history down
|
|
60
|
+
* to an empty array is stale, and the caller resends the full history rather
|
|
54
61
|
* than handing the model an empty payload.
|
|
55
62
|
*/
|
|
56
|
-
export declare function decideCachedHistorySlice(savedCount: number,
|
|
63
|
+
export declare function decideCachedHistorySlice(savedCount: number, history: HistoryMessage[]): HistorySliceDecision;
|
|
57
64
|
export {};
|
|
@@ -30,29 +30,31 @@ export function shouldCommitCachedTurn(context) {
|
|
|
30
30
|
* attachment probing). Kept here so the decision can be unit-tested in
|
|
31
31
|
* isolation.
|
|
32
32
|
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
33
|
+
* A non-zero `savedCount` is itself the proof that a cache holds a rendered
|
|
34
|
+
* prefix: it is recorded only after a save is verified. It means the prefix
|
|
35
|
+
* covers that many messages, so slicing from it drops exactly what the cache
|
|
36
|
+
* supplies — the system message among them. Without a usable boundary the
|
|
37
|
+
* cache supplies nothing, so the whole history goes to the addon, system
|
|
38
|
+
* message included: anything held back would simply be missing from the
|
|
39
|
+
* prompt.
|
|
40
|
+
*
|
|
41
|
+
* The regression guard: a non-zero `savedCount` that slices the history down
|
|
42
|
+
* to an empty array is stale, and the caller resends the full history rather
|
|
36
43
|
* than handing the model an empty payload.
|
|
37
44
|
*/
|
|
38
|
-
export function decideCachedHistorySlice(savedCount,
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
messages: history.filter((msg) => msg.role !== 'system'),
|
|
42
|
-
clearStaleCount: false
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
const canSlice = savedCount > 0 && savedCount <= history.length;
|
|
46
|
-
const sliced = canSlice ? history.slice(savedCount) : null;
|
|
45
|
+
export function decideCachedHistorySlice(savedCount, history) {
|
|
46
|
+
const hasCachedPrefix = savedCount > 0 && history.length > 0;
|
|
47
|
+
const sliced = hasCachedPrefix && savedCount <= history.length ? history.slice(savedCount) : null;
|
|
47
48
|
// A non-null slice that is empty means the saved count is stale: the
|
|
48
49
|
// cached turn boundary is claiming the entire current history is
|
|
49
50
|
// already cached, which happens when a previous turn was cancelled
|
|
50
51
|
// mid-decode and still recorded `history.length + 1`. Treat it as a
|
|
51
|
-
// bad state and resend the full
|
|
52
|
+
// bad state and resend the full history.
|
|
52
53
|
const useSlice = sliced !== null && sliced.length > 0;
|
|
53
|
-
const messages = useSlice ? sliced : history.filter((msg) => msg.role !== 'system');
|
|
54
54
|
return {
|
|
55
|
-
messages,
|
|
56
|
-
|
|
55
|
+
messages: useSlice ? sliced : history,
|
|
56
|
+
// Only a boundary that was actually consulted can be stale. Absent a
|
|
57
|
+
// history to slice, the count is simply unused and left alone.
|
|
58
|
+
clearStaleCount: hasCachedPrefix && !useSlice
|
|
57
59
|
};
|
|
58
60
|
}
|