@qvac/inference 0.19.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/NOTICE +43 -11
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/index.d.ts +2 -1
  5. package/dist/api/index.js +2 -1
  6. package/dist/api/text-to-speech.d.ts +31 -4
  7. package/dist/api/text-to-speech.js +117 -21
  8. package/dist/api/vector-index.d.ts +142 -0
  9. package/dist/api/vector-index.js +168 -0
  10. package/dist/api/world.d.ts +2 -2
  11. package/dist/errors/index.d.ts +12 -0
  12. package/dist/errors/index.js +21 -0
  13. package/dist/handlers/vector-index.d.ts +2 -0
  14. package/dist/handlers/vector-index.js +40 -0
  15. package/dist/index.d.ts +1 -1
  16. package/dist/index.js +1 -1
  17. package/dist/models/registry/models.d.ts +351 -12
  18. package/dist/models/registry/models.js +2271 -1957
  19. package/dist/models/registry/resource-profiles.js +135 -0
  20. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  21. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  22. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  30. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  31. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  32. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  33. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  34. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  35. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  36. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  37. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  38. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  40. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  41. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  43. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  44. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  45. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
  46. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
  47. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  49. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
  50. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  51. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  52. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  53. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  54. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  55. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  56. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  57. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  58. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  59. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  60. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  61. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  62. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  63. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  64. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  65. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  66. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  67. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  68. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  70. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  71. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  72. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  73. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  74. package/dist/plugins/index.d.ts +1 -1
  75. package/dist/plugins/index.js +1 -1
  76. package/dist/plugins/ops/load-model.js +22 -2
  77. package/dist/plugins/ops/translate.js +58 -12
  78. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  79. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  80. package/dist/plugins/registry.d.ts +15 -0
  81. package/dist/plugins/registry.js +133 -15
  82. package/dist/plugins/turbovec-backend.d.ts +17 -0
  83. package/dist/plugins/turbovec-backend.js +99 -0
  84. package/dist/registry.js +4 -0
  85. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  86. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  87. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  88. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  89. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  90. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  91. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  92. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  93. package/dist/runtime/env.d.ts +2 -0
  94. package/dist/runtime/env.js +9 -2
  95. package/dist/runtime/lifecycle.js +2 -0
  96. package/dist/runtime/model-registry.d.ts +3 -1
  97. package/dist/runtime/model-registry.js +2 -1
  98. package/dist/runtime/request-context.js +17 -0
  99. package/dist/runtime/vector-index-backend.d.ts +25 -0
  100. package/dist/runtime/vector-index-backend.js +1 -0
  101. package/dist/runtime/vector-index-registry.d.ts +76 -0
  102. package/dist/runtime/vector-index-registry.js +161 -0
  103. package/dist/schemas/assess-model-fit.d.ts +44 -0
  104. package/dist/schemas/assess-model-fit.js +40 -0
  105. package/dist/schemas/audio-gen.d.ts +479 -4
  106. package/dist/schemas/audio-gen.js +376 -17
  107. package/dist/schemas/batch-completion-stream.d.ts +7 -0
  108. package/dist/schemas/batch-completion-stream.js +5 -3
  109. package/dist/schemas/common.d.ts +749 -15
  110. package/dist/schemas/common.js +8 -1
  111. package/dist/schemas/completion-event.d.ts +11 -1
  112. package/dist/schemas/completion-event.js +3 -0
  113. package/dist/schemas/completion-stream.d.ts +20 -0
  114. package/dist/schemas/completion-stream.js +41 -4
  115. package/dist/schemas/config.d.ts +48 -30
  116. package/dist/schemas/errors.d.ts +4 -0
  117. package/dist/schemas/errors.js +22 -0
  118. package/dist/schemas/index.d.ts +1 -0
  119. package/dist/schemas/index.js +1 -0
  120. package/dist/schemas/llamacpp-config.d.ts +4 -8
  121. package/dist/schemas/llamacpp-config.js +4 -4
  122. package/dist/schemas/load-model.d.ts +2503 -139
  123. package/dist/schemas/model-config-schemas.d.ts +228 -5
  124. package/dist/schemas/plugin.d.ts +10 -0
  125. package/dist/schemas/sdcpp-config.d.ts +221 -4
  126. package/dist/schemas/sdcpp-config.js +128 -22
  127. package/dist/schemas/text-to-speech.d.ts +780 -19
  128. package/dist/schemas/text-to-speech.js +298 -44
  129. package/dist/schemas/transcription-config.js +7 -4
  130. package/dist/schemas/vector-index.d.ts +194 -0
  131. package/dist/schemas/vector-index.js +185 -0
  132. package/dist/utils/addon-responses.d.ts +29 -3
  133. package/dist/utils/aggregate-events.d.ts +3 -1
  134. package/dist/utils/aggregate-events.js +7 -1
  135. package/dist/utils/tts-stats.d.ts +15 -11
  136. package/dist/utils/tts-stats.js +33 -11
  137. package/package.json +28 -25
@@ -2,18 +2,42 @@ import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
2
2
  import { type CacheMessage } from '../../../../utils/index.js';
3
3
  import type { Logger } from '../../../../logging/types.js';
4
4
  import { type AbortSignal } from 'bare-abort-controller';
5
+ import { z } from 'zod';
6
+ /**
7
+ * Coordinates six KV-cache state layers:
8
+ *
9
+ * 1. `cachedPrefixes` — saved message boundaries, and whether a tool
10
+ * block was rendered into them.
11
+ * 2. `initializedCaches` — caches this process has established.
12
+ * 3. On-disk `.bin` files written by the addon.
13
+ * 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
14
+ * boundary outlives the process; see `readPrefixSidecar`.
15
+ * 5. `activeCachePaths` — per-path refs that block in-flight eviction.
16
+ * 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
17
+ *
18
+ * Every turn must finish through `commitTurn`, `rollback`, or the
19
+ * non-destructive `releaseTurn` so all inference state stays aligned,
20
+ * the active-path ref is released, and marker metadata follows the
21
+ * cache directory lifecycle.
22
+ */
23
+ /** What the kv-cache file at a given path is known to hold. */
24
+ declare const cachedPrefixSchema: z.ZodObject<{
25
+ messages: z.ZodNumber;
26
+ toolBlock: z.ZodBoolean;
27
+ }, z.core.$strip>;
28
+ type CachedPrefix = z.infer<typeof cachedPrefixSchema>;
5
29
  export interface TurnHandle {
6
30
  /** Resolved on-disk cache file path the addon will read from / write to. */
7
31
  readonly cachePath: string;
8
32
  /**
9
33
  * Snapshot of the on-disk saved-message count at `beginTurn` time
10
- * (0 if the cache was just primed). Consumed by `decideCachedHistorySlice`
11
- * to pick the message tail for the next addon call.
34
+ * (0 on a cold turn). Consumed by `decideCachedHistorySlice` to pick the
35
+ * message tail for the next addon call.
12
36
  */
13
37
  readonly savedCount: number;
14
38
  /**
15
39
  * Whether the cached prefix already holds a rendered static tool block, so
16
- * this turn can leave it out of its payload. False on a fresh prime and
40
+ * this turn can leave it out of its payload. False on a cold turn and
17
41
  * whenever the previous turn couldn't confirm the block reached the model.
18
42
  */
19
43
  readonly toolBlockCached: boolean;
@@ -24,14 +48,6 @@ export interface BeginCustomTurnInput {
24
48
  customKey: string;
25
49
  /** Hash of system prompt + complete tool definitions. */
26
50
  configHash: string;
27
- /**
28
- * Prime the cache by sending the system prompt to the addon. Tools are not
29
- * primed — a prefix with no user turn is not a renderable conversation for
30
- * every template — so they travel with a turn instead. Called when the cache
31
- * doesn't exist in-memory OR on disk. Kept as an injected closure so this
32
- * module has no dependency on the model registry / addon.
33
- */
34
- primeIfMissing: (cachePath: string) => Promise<void>;
35
51
  /**
36
52
  * Request abort signal. When it aborts while this turn is queued behind a
37
53
  * same-file peer's write lock, the wait is abandoned so the request's scope
@@ -45,8 +61,6 @@ export interface BeginAutoTurnInput {
45
61
  configHash: string;
46
62
  /** Conversation history used to compute the pre-response cache key. */
47
63
  history: CacheMessage[];
48
- /** See `BeginCustomTurnInput.primeIfMissing`. */
49
- primeIfMissing: (cachePath: string) => Promise<void>;
50
64
  /** See `BeginCustomTurnInput.signal`. */
51
65
  signal?: AbortSignal;
52
66
  }
@@ -80,10 +94,10 @@ export type CommitResult = StaticCommitResult | AutoRenameCommitResult;
80
94
  export interface KvCacheSession {
81
95
  /**
82
96
  * Open a new turn against the cache. Resolves the cache file path,
83
- * primes the system-prompt cache if needed (delegated to
84
- * `input.primeIfMissing`), marks the cache initialized, and returns a
85
- * `TurnHandle` the handler attaches to `ctx.scope.defer(...)` for the
86
- * rollback hook. Auto-cache path resolution is serialized with
97
+ * ensures its parent directory exists, and returns a `TurnHandle` the
98
+ * handler attaches to `ctx.scope.defer(...)` for the rollback hook. The
99
+ * addon's own `saveCacheToDisk` is what writes the file, so a cold turn
100
+ * opens without one. Auto-cache path resolution is serialized with
87
101
  * retention deletion before the handle is returned.
88
102
  */
89
103
  beginTurn(input: BeginTurnInput): Promise<TurnHandle>;
@@ -109,17 +123,17 @@ export interface KvCacheSession {
109
123
  /**
110
124
  * Non-destructive counterpart of `rollback`: releases locks and refs but
111
125
  * keeps the committed disk cache and its recorded prefix valid for a retry.
112
- * A cache freshly primed by this same turn rolls back instead.
126
+ * A cache this same turn created rolls back instead.
113
127
  */
114
128
  releaseTurn(turn: TurnHandle): Promise<void>;
115
129
  /**
116
- * Forget the in-memory saved-message count for the turn's path
117
- * without unlinking the file or clearing the init flag. Used when
118
- * `decideCachedHistorySlice` detects a stale boundary
130
+ * Forget the saved-message count for the turn's path, in memory and in the
131
+ * sidecar, without unlinking the cache file or clearing the init flag. Used
132
+ * when `decideCachedHistorySlice` detects a stale boundary
119
133
  * (`clearStaleCount: true`) — the next turn re-sends the full history
120
134
  * but the cache itself is still usable.
121
135
  */
122
- dropStaleSavedCount(turn: TurnHandle): void;
136
+ dropStaleSavedCount(turn: TurnHandle): Promise<void>;
123
137
  }
124
138
  /**
125
139
  * Construct a session bound to one `(modelId, turn-owning request)`
@@ -160,8 +174,8 @@ export declare function createKvCacheSession(modelId: string, options?: {
160
174
  * cleared without a lock, so an interleaving that lands the delete's
161
175
  * in-memory cleanup after a concurrent turn has already renamed/committed
162
176
  * its file splits state: the file stays on disk while its saved-count and
163
- * init flag are cleared, so the next turn sees the file, skips priming, and
164
- * reports `savedCount=0`. Callers must not delete a key that is in active use.
177
+ * init flag are cleared, so the next turn loads the file and reports
178
+ * `savedCount=0`. Callers must not delete a key that is in active use.
165
179
  * The `auto` target is the exception: it skips keys an in-flight turn holds
166
180
  * and clears each one under the cache-state lock.
167
181
  */
@@ -186,6 +200,8 @@ export declare const __kvCacheSessionTestHooks: {
186
200
  getSavedCount(cachePath: string): number | undefined;
187
201
  setSavedCountForTest(cachePath: string, count: number): void;
188
202
  getToolBlockCachedForTest(cachePath: string): boolean;
203
+ getPrefixSidecarPathForTest(cachePath: string): string;
204
+ readPrefixSidecarForTest(cachePath: string): Promise<CachedPrefix | null>;
189
205
  hasInitializedPath(cachePath: string): boolean;
190
206
  getActivePathCountForTest(cachePath: string): number;
191
207
  getLastAutoCacheSweepMsForTest(): number;
@@ -6,27 +6,130 @@ import { isMobile } from '../../../../runtime/state.js';
6
6
  import { getKVCacheDir } from '../../../../utils/index.js';
7
7
  import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
8
8
  import { getEngineLogger } from '../../../../logging/index.js';
9
+ import { z } from 'zod';
9
10
  // Used by cross-model paths that have no `RequestContext` (e.g.
10
11
  // `deleteKvCacheState`). Per-session call sites receive a logger from
11
12
  // the caller — typically `withRequestContext(...)`.
12
13
  const moduleLogger = getEngineLogger();
14
+ /**
15
+ * Coordinates six KV-cache state layers:
16
+ *
17
+ * 1. `cachedPrefixes` — saved message boundaries, and whether a tool
18
+ * block was rendered into them.
19
+ * 2. `initializedCaches` — caches this process has established.
20
+ * 3. On-disk `.bin` files written by the addon.
21
+ * 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
22
+ * boundary outlives the process; see `readPrefixSidecar`.
23
+ * 5. `activeCachePaths` — per-path refs that block in-flight eviction.
24
+ * 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
25
+ *
26
+ * Every turn must finish through `commitTurn`, `rollback`, or the
27
+ * non-destructive `releaseTurn` so all inference state stays aligned,
28
+ * the active-path ref is released, and marker metadata follows the
29
+ * cache directory lifecycle.
30
+ */
31
+ // ----- module-scoped state. The session is the single mutation point
32
+ // for the in-memory KV-cache bookkeeping. -----
33
+ /** What the kv-cache file at a given path is known to hold. */
34
+ const cachedPrefixSchema = z.object({
35
+ /** Number of chat messages the file on disk is known to cover. */
36
+ messages: z.number().int().nonnegative(),
37
+ /**
38
+ * Whether a static tool block was rendered into that prefix. Tracked
39
+ * rather than inferred from `messages`, because a committed turn is not
40
+ * proof that its tool block reached the model: the addon drops tools and
41
+ * still returns a usable prompt when the chat template rejects them.
42
+ */
43
+ toolBlock: z.boolean()
44
+ });
45
+ // The sidecar is written after the addon has already saved the `.bin`, so a
46
+ // crash between the two leaves a boundary that describes an older file. The
47
+ // `.bin` size is a function of the tokens it holds: it is unchanged when the
48
+ // addon re-saves the same state (cancel rewind, key switch) and larger once
49
+ // another turn landed in it, which is the case the fingerprint must catch.
50
+ const persistedPrefixSchema = cachedPrefixSchema.extend({
51
+ binSize: z.number().int().nonnegative()
52
+ });
13
53
  /**
14
54
  * What the kv-cache file on disk is known to cover, keyed by cache path.
15
- * Written by `commitTurn`, read by `getSavedCount`, deleted by `rollback` /
16
- * `delete` / `dropStaleSavedCount`. The same INVARIANT that existed in
17
- * `kv-cache-state.ts` still holds: an entry is present only when the
18
- * corresponding `.bin` file is considered trustworthy. Cancelled or
19
- * zero-token turns must remove the entry so the next-turn slice doesn't read
20
- * a stale boundary.
55
+ * Written by `commitTurn`, deleted by `rollback`, `delete`,
56
+ * `dropStaleSavedCount` and the orphan clear in `beginTurn`. The same
57
+ * INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
58
+ * present only when the corresponding `.bin` file is considered trustworthy,
59
+ * so any turn that leaves the file holding something the boundary does not
60
+ * describe has to remove it.
61
+ *
62
+ * This map outranks the sidecar. Both are written by the same commit, so an
63
+ * entry here is never the older copy, and `restorePersistedCache` reads the
64
+ * file only to repopulate an empty map after a restart.
21
65
  */
22
66
  const cachedPrefixes = new Map();
67
+ const PREFIX_SIDECAR_SUFFIX = '.meta.json';
68
+ function prefixSidecarPath(cachePath) {
69
+ return `${cachePath}${PREFIX_SIDECAR_SUFFIX}`;
70
+ }
71
+ async function writePrefixSidecar(cachePath, prefix, logger) {
72
+ try {
73
+ const { size } = await fsPromises.stat(cachePath);
74
+ await fsPromises.writeFile(prefixSidecarPath(cachePath), JSON.stringify({ ...prefix, binSize: size }));
75
+ }
76
+ catch (error) {
77
+ // Only the file goes: a sidecar left from the previous commit describes a
78
+ // shorter `.bin` than the one now on disk. The in-memory boundary is the
79
+ // one this process keeps slicing against and stays.
80
+ await removePrefixSidecar(cachePath);
81
+ logger.warn(`[kv-cache] Failed to persist saved-message boundary; this process stays warm but a restart will start the cache cold. path=${cachePath} error=${error instanceof Error ? error.message : String(error)}`);
82
+ }
83
+ }
84
+ // Anything unreadable, malformed, or describing a `.bin` of a different size
85
+ // is discarded: the cache is then used from a cold boundary, which is the
86
+ // pre-sidecar behaviour. Only ever consulted for a path `cachedPrefixes` has
87
+ // no entry for, so it can never contradict a live boundary. Never throws: the
88
+ // caller holds the cache-path write lock and a `.bin` can vanish under it
89
+ // (`deleteKvCacheState` takes no locks).
90
+ async function readPrefixSidecar(cachePath) {
91
+ let parsed;
92
+ let binSize;
93
+ try {
94
+ const raw = await fsPromises.readFile(prefixSidecarPath(cachePath), 'utf8');
95
+ const result = persistedPrefixSchema.safeParse(JSON.parse(raw));
96
+ if (!result.success)
97
+ return null;
98
+ parsed = result.data;
99
+ binSize = (await fsPromises.stat(cachePath)).size;
100
+ }
101
+ catch {
102
+ return null;
103
+ }
104
+ if (binSize !== parsed.binSize) {
105
+ // The `.bin` changed after this boundary was recorded, so the file is
106
+ // stale whatever wrote it. Only the file: the caller guarantees the map
107
+ // holds nothing for this path.
108
+ await removePrefixSidecar(cachePath);
109
+ return null;
110
+ }
111
+ return { messages: parsed.messages, toolBlock: parsed.toolBlock };
112
+ }
113
+ /** Drops the persisted boundary, leaving whatever is in memory alone. */
114
+ async function removePrefixSidecar(cachePath) {
115
+ try {
116
+ await fsPromises.unlink(prefixSidecarPath(cachePath));
117
+ }
118
+ catch {
119
+ // No sidecar for this path.
120
+ }
121
+ }
122
+ /** Drops the boundary everywhere: the cache it describes is gone or untrusted. */
123
+ async function forgetPrefix(cachePath) {
124
+ cachedPrefixes.delete(cachePath);
125
+ await removePrefixSidecar(cachePath);
126
+ }
23
127
  /**
24
- * In-memory registry of caches initialized this session. The addon
25
- * defers disk writes, so the absence of a `.bin` file on disk isn't
26
- * proof that the cache hasn't been primed in this process. Keyed
27
- * by the resolved cache path, so aliased keys that name one file share an
28
- * entry and on-disk caches from older process runs still hit the lazy-load
29
- * path in `beginTurn`.
128
+ * In-memory registry of caches established this session, recorded once a save
129
+ * is verified. The addon defers disk writes, so the absence of a `.bin` file on
130
+ * disk isn't proof that no cache exists for this process. Keyed by the resolved
131
+ * cache path, so aliased keys that name one file share an entry and on-disk
132
+ * caches from older process runs still hit the lazy-load path in `beginTurn`.
30
133
  */
31
134
  const initializedCaches = new Set();
32
135
  const activeCachePaths = new Map();
@@ -215,7 +318,7 @@ async function deleteInactiveAutoCaches() {
215
318
  const cacheKeys = await planAutoCacheEvictions({
216
319
  activeCachePaths: snapshotActivePaths(),
217
320
  // The quota pass stops at `retainedBytes <= maxBytes`, and a total reaches
218
- // 0 while zero-byte caches remain (an empty `.bin` from a crashed prime).
321
+ // 0 while zero-byte caches remain (an empty `.bin` from an interrupted save).
219
322
  // A negative quota is unreachable, so every inactive entry is selected.
220
323
  maxBytes: -1,
221
324
  // No age threshold: the quota pass above already selects everything.
@@ -254,7 +357,7 @@ export function createKvCacheSession(modelId, options) {
254
357
  releaseWriteLock,
255
358
  committed: false,
256
359
  rolledBack: false,
257
- freshlyPrimed: false
360
+ createdByThisTurn: false
258
361
  });
259
362
  markCachePathActive(cachePath);
260
363
  return handle;
@@ -276,7 +379,6 @@ export function createKvCacheSession(modelId, options) {
276
379
  }
277
380
  throw error;
278
381
  }
279
- // A turn cancelled by the time it holds the lock must not prime (native work).
280
382
  // getCacheFilePath already mkdir'd the parent, so prune it before surfacing
281
383
  // the cancellation the plugin rides.
282
384
  if (input.signal?.aborted) {
@@ -284,36 +386,25 @@ export function createKvCacheSession(modelId, options) {
284
386
  releaseWriteLock();
285
387
  throw new CacheLockAbortError(input.signal.reason);
286
388
  }
389
+ // In-memory registry check first — the addon defers disk writes, so a
390
+ // just-saved cache may not yet exist on disk. If the in-memory flag isn't
391
+ // set, fall back to a filesystem probe so caches surviving across process
392
+ // restarts still hit the reuse path. Resolved before the handle is made so
393
+ // it snapshots the restored boundary.
394
+ let exists = initializedCaches.has(cachePath);
395
+ if (!exists)
396
+ exists = await restorePersistedCache(cachePath);
397
+ // A boundary without its `.bin` describes a cache that no longer exists.
398
+ if (!exists)
399
+ await forgetPrefix(cachePath);
287
400
  const handle = makeHandle(cachePath, undefined, releaseWriteLock, input.signal);
288
401
  try {
289
- // In-memory registry check first — the addon defers disk writes, so
290
- // a freshly-primed cache may not yet exist on disk. If the
291
- // in-memory flag isn't set, fall back to a filesystem probe so
292
- // caches surviving across process restarts still hit the reuse path.
293
- let exists = initializedCaches.has(cachePath);
294
- if (!exists) {
295
- try {
296
- await fsPromises.access(cachePath);
297
- exists = true;
298
- initializedCaches.add(cachePath);
299
- }
300
- catch {
301
- exists = false;
302
- }
303
- }
304
402
  logCacheStatus(input.customKey, exists);
305
403
  if (!exists) {
306
- // Recreate the parent dir if a same-key peer's rollback pruned it after our lock wait.
404
+ // Recreate the parent dir if a same-key peer's rollback pruned it after
405
+ // our lock wait, so the addon's own save has somewhere to land.
307
406
  await fsPromises.mkdir(path.dirname(cachePath), { recursive: true });
308
- // The access probe / mkdir above yielded, so a cancel may have landed
309
- // since the acquire-time check — re-check right before native priming.
310
- // The catch below prunes the directory just created.
311
- if (input.signal?.aborted)
312
- throw new CacheLockAbortError(input.signal.reason);
313
- await input.primeIfMissing(cachePath);
314
- await verifyPrimedFile(cachePath, logger);
315
- initializedCaches.add(cachePath);
316
- turnState.get(handle).freshlyPrimed = true;
407
+ turnState.get(handle).createdByThisTurn = true;
317
408
  }
318
409
  return handle;
319
410
  }
@@ -351,7 +442,6 @@ export function createKvCacheSession(modelId, options) {
351
442
  }
352
443
  throw error;
353
444
  }
354
- // A turn cancelled by the time it holds the lock must not prime (native work).
355
445
  // Discovery already mkdir'd the parent and wrote the auto marker; clean both
356
446
  // before surfacing the cancellation.
357
447
  if (input.signal?.aborted) {
@@ -389,17 +479,10 @@ export function createKvCacheSession(modelId, options) {
389
479
  logCacheStatus('auto', cacheExists);
390
480
  try {
391
481
  if (!cacheExists) {
392
- // Recreate the parent dir if a same-file peer's rename pruned it.
482
+ // Recreate the parent dir if a same-file peer's rename pruned it, so
483
+ // the addon's own save has somewhere to land.
393
484
  await fsPromises.mkdir(path.dirname(cachePath), { recursive: true });
394
- // The discovery / mkdir above yielded, so a cancel may have landed since
395
- // the acquire-time check — re-check right before native priming. The
396
- // catch below prunes the directory and auto marker.
397
- if (input.signal?.aborted)
398
- throw new CacheLockAbortError(input.signal.reason);
399
- await input.primeIfMissing(cachePath);
400
- await verifyPrimedFile(cachePath, logger);
401
- initializedCaches.add(cachePath);
402
- turnState.get(handle).freshlyPrimed = true;
485
+ turnState.get(handle).createdByThisTurn = true;
403
486
  }
404
487
  return handle;
405
488
  }
@@ -432,13 +515,13 @@ export function createKvCacheSession(modelId, options) {
432
515
  // at the same path. Verify the file persisted (the addon
433
516
  // currently swallows save errors — see TODO in
434
517
  // `verifySaveAndRecord`) and record the new boundary.
435
- const ok = await verifySaveAndRecord(state.cachePath, result.messageCount, result.toolBlockCached);
518
+ const prefix = { messages: result.messageCount, toolBlock: result.toolBlockCached };
519
+ const ok = await verifySaveAndRecord(state.cachePath, prefix);
436
520
  if (!ok) {
437
- // The expected save didn't land — treat the turn as a rollback
438
- // so the next turn re-primes cleanly.
439
521
  await runRollback(state);
440
522
  return;
441
523
  }
524
+ await writePrefixSidecar(state.cachePath, prefix, logger);
442
525
  state.committed = true;
443
526
  releaseCachePath(state.cachePath);
444
527
  state.releaseWriteLock();
@@ -481,11 +564,11 @@ export function createKvCacheSession(modelId, options) {
481
564
  releaseCachePath(sourceCachePath);
482
565
  state.cachePath = result.targetCachePath;
483
566
  state.autoCacheKey = targetCacheKey;
567
+ await forgetPrefix(sourceCachePath);
484
568
  await pruneEmptyCacheDirectories(sourceCachePath, snapshotActivePaths());
485
569
  if (sourceCacheKey !== undefined) {
486
570
  await removeAutoCacheMarkerIfMissing(sourceCacheKey);
487
571
  }
488
- cachedPrefixes.delete(sourceCachePath);
489
572
  // state.cachePath was just reassigned to the target; clear the SOURCE
490
573
  // entry, not the freshly-valid target.
491
574
  initializedCaches.delete(sourceCachePath);
@@ -496,7 +579,10 @@ export function createKvCacheSession(modelId, options) {
496
579
  await runRollback(state);
497
580
  return;
498
581
  }
499
- const ok = await verifySaveAndRecord(result.targetCachePath, result.messageCount, result.toolBlockCached);
582
+ const ok = await verifySaveAndRecord(result.targetCachePath, {
583
+ messages: result.messageCount,
584
+ toolBlock: result.toolBlockCached
585
+ });
500
586
  if (!ok) {
501
587
  // Rename succeeded but the file isn't where we expected. Roll back via
502
588
  // the target path instead of the (now-empty) source.
@@ -529,9 +615,8 @@ export function createKvCacheSession(modelId, options) {
529
615
  return;
530
616
  if (state.committed || state.rolledBack)
531
617
  return;
532
- // A cache this same turn primed has no committed state to keep — a failed
533
- // first turn must not leave its own prime behind.
534
- if (state.freshlyPrimed) {
618
+ // Nothing committed exists to keep, so a failed first turn is destructive.
619
+ if (state.createdByThisTurn) {
535
620
  await runRollback(state);
536
621
  return;
537
622
  }
@@ -550,8 +635,13 @@ export function createKvCacheSession(modelId, options) {
550
635
  await fsPromises.unlink(state.cachePath);
551
636
  }
552
637
  catch (unlinkError) {
553
- logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
638
+ // A turn that fails before the addon's save has no file to remove, so
639
+ // ENOENT leaves no stale state behind and is not worth a warning.
640
+ if (unlinkError.code !== 'ENOENT') {
641
+ logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
642
+ }
554
643
  }
644
+ await forgetPrefix(state.cachePath);
555
645
  // Release before pruning so an empty parent can go; a sibling still holding
556
646
  // the path keeps it in the active snapshot and protects the directory.
557
647
  releaseCachePath(state.cachePath);
@@ -560,17 +650,16 @@ export function createKvCacheSession(modelId, options) {
560
650
  await removeAutoCacheMarkerIfMissing(state.autoCacheKey);
561
651
  }
562
652
  initializedCaches.delete(state.cachePath);
563
- cachedPrefixes.delete(state.cachePath);
564
653
  state.rolledBack = true;
565
654
  state.releaseWriteLock();
566
655
  if (state.autoCacheKey !== undefined)
567
656
  scheduleAutoCacheSweep(logger);
568
657
  }
569
- function dropStaleSavedCount(turn) {
658
+ async function dropStaleSavedCount(turn) {
570
659
  const state = turnState.get(turn);
571
660
  if (!state)
572
661
  return;
573
- cachedPrefixes.delete(state.cachePath);
662
+ await forgetPrefix(state.cachePath);
574
663
  }
575
664
  return {
576
665
  beginTurn,
@@ -611,8 +700,8 @@ export function createKvCacheSession(modelId, options) {
611
700
  * cleared without a lock, so an interleaving that lands the delete's
612
701
  * in-memory cleanup after a concurrent turn has already renamed/committed
613
702
  * its file splits state: the file stays on disk while its saved-count and
614
- * init flag are cleared, so the next turn sees the file, skips priming, and
615
- * reports `savedCount=0`. Callers must not delete a key that is in active use.
703
+ * init flag are cleared, so the next turn loads the file and reports
704
+ * `savedCount=0`. Callers must not delete a key that is in active use.
616
705
  * The `auto` target is the exception: it skips keys an in-flight turn holds
617
706
  * and clears each one under the cache-state lock.
618
707
  */
@@ -649,62 +738,6 @@ export async function deleteKvCacheState(target) {
649
738
  clearInitializedCachesByPrefix(removedPath, path.sep);
650
739
  }
651
740
  // ----- private helpers -----
652
- /**
653
- * Verify that the addon actually persisted a usable cache file after a
654
- * prime. Mirrors the `verifySaveAndRecord` access-probe used at commit
655
- * time, applied at prime time so the session doesn't mark a cache
656
- * `initializedCaches.add(...)` against a path that's missing or empty
657
- * on disk.
658
- *
659
- * Failure modes this catches:
660
- *
661
- * - The addon's `model.run({ saveSessionPath })` was interrupted
662
- * before the save call ran (e.g. signal abort during prefill); the
663
- * prime closure resolves cleanly because addon save errors are not
664
- * propagated, but no file is on disk.
665
- * - The addon's `llama_state_save_file` was called but produced an
666
- * empty file (out-of-space / fs error swallowed by the addon).
667
- *
668
- * Failure modes this does **NOT** catch:
669
- *
670
- * - A partial-but-nonzero file written by the addon (e.g. header +
671
- * truncated KV state). Catching this requires either an
672
- * addon-side change (have `CacheManager::writeCacheFile` check the
673
- * return value of `llama_state_save_file` and throw on failure) or
674
- * a structural hash check we can't currently compute
675
- * engine-side. Filed as a follow-up — see `cache-api.md` in the addon
676
- * repo / tracking ticket.
677
- *
678
- * On failure we best-effort `unlink` an empty leftover file (so the
679
- * next existence probe doesn't trust it) and throw — the handler in
680
- * `completion-stream.ts` lets the error propagate up and no
681
- * `initializedCaches` entry is recorded.
682
- */
683
- async function verifyPrimedFile(cachePath, logger) {
684
- let stats;
685
- try {
686
- stats = await fsPromises.stat(cachePath);
687
- }
688
- catch (statError) {
689
- // ENOENT is the common case here — addon prime returned without
690
- // calling save (most often: signal abort during prefill).
691
- await pruneEmptyCacheDirectories(cachePath, snapshotActivePaths());
692
- throw new Error(`[kv-cache] prime closure resolved but no cache file was written. path=${cachePath} cause=${statError instanceof Error ? statError.message : String(statError)}`);
693
- }
694
- if (stats.size === 0) {
695
- // Best-effort cleanup so a future probe doesn't trust the empty
696
- // file. Unlink failure is non-fatal — we still throw on the
697
- // primary "prime didn't persist" condition.
698
- try {
699
- await fsPromises.unlink(cachePath);
700
- await pruneEmptyCacheDirectories(cachePath, snapshotActivePaths());
701
- }
702
- catch (unlinkError) {
703
- logger.warn(`[kv-cache] Failed to remove empty primed cache file. path=${cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
704
- }
705
- throw new Error(`[kv-cache] prime closure resolved but cache file is empty. path=${cachePath}`);
706
- }
707
- }
708
741
  /**
709
742
  * Verify the addon actually persisted the cache file before recording
710
743
  * its message count. The addon currently swallows write errors
@@ -716,10 +749,11 @@ async function verifyPrimedFile(cachePath, logger) {
716
749
  * false), drop the `access()` probe and wrap the `model.run()` call in
717
750
  * a real try/catch that forwards the error.
718
751
  */
719
- async function verifySaveAndRecord(cachePath, messageCount, toolBlockCached) {
752
+ async function verifySaveAndRecord(cachePath, prefix) {
720
753
  try {
721
754
  await fsPromises.access(cachePath);
722
- cachedPrefixes.set(cachePath, { messages: messageCount, toolBlock: toolBlockCached });
755
+ cachedPrefixes.set(cachePath, prefix);
756
+ initializedCaches.add(cachePath);
723
757
  return true;
724
758
  }
725
759
  catch (err) {
@@ -728,6 +762,30 @@ async function verifySaveAndRecord(cachePath, messageCount, toolBlockCached) {
728
762
  return false;
729
763
  }
730
764
  }
765
+ /**
766
+ * Adopt a `.bin` left by an earlier process run: record it as established and
767
+ * pick up the boundary committed alongside it. Returns whether the file exists.
768
+ *
769
+ * An auto-rename commits its target into `cachedPrefixes` without an init
770
+ * flag, so this can run for a path the map already knows. The map wins there
771
+ * and the sidecar is left unread, which keeps the two from ever having to be
772
+ * reconciled.
773
+ */
774
+ async function restorePersistedCache(cachePath) {
775
+ try {
776
+ await fsPromises.access(cachePath);
777
+ }
778
+ catch {
779
+ return false;
780
+ }
781
+ initializedCaches.add(cachePath);
782
+ if (cachedPrefixes.has(cachePath))
783
+ return true;
784
+ const prefix = await readPrefixSidecar(cachePath);
785
+ if (prefix !== null)
786
+ cachedPrefixes.set(cachePath, prefix);
787
+ return true;
788
+ }
731
789
  function clearCachedMessageCountsByPrefix(prefix, sep) {
732
790
  if (!prefix) {
733
791
  cachedPrefixes.clear();
@@ -773,6 +831,12 @@ export const __kvCacheSessionTestHooks = {
773
831
  getToolBlockCachedForTest(cachePath) {
774
832
  return cachedPrefixes.get(cachePath)?.toolBlock ?? false;
775
833
  },
834
+ getPrefixSidecarPathForTest(cachePath) {
835
+ return prefixSidecarPath(cachePath);
836
+ },
837
+ readPrefixSidecarForTest(cachePath) {
838
+ return readPrefixSidecar(cachePath);
839
+ },
776
840
  hasInitializedPath(cachePath) {
777
841
  return initializedCaches.has(cachePath);
778
842
  },
@@ -48,10 +48,17 @@ export declare function shouldCommitCachedTurn(context: CacheCommitContext): boo
48
48
  * attachment probing). Kept here so the decision can be unit-tested in
49
49
  * isolation.
50
50
  *
51
- * The key regression guard: when a non-zero `savedCount` would slice
52
- * the history down to an empty array, it is treated as stale — the
53
- * caller falls back to sending the system-stripped full history rather
51
+ * A non-zero `savedCount` is itself the proof that a cache holds a rendered
52
+ * prefix: it is recorded only after a save is verified. It means the prefix
53
+ * covers that many messages, so slicing from it drops exactly what the cache
54
+ * supplies — the system message among them. Without a usable boundary the
55
+ * cache supplies nothing, so the whole history goes to the addon, system
56
+ * message included: anything held back would simply be missing from the
57
+ * prompt.
58
+ *
59
+ * The regression guard: a non-zero `savedCount` that slices the history down
60
+ * to an empty array is stale, and the caller resends the full history rather
54
61
  * than handing the model an empty payload.
55
62
  */
56
- export declare function decideCachedHistorySlice(savedCount: number, cacheExists: boolean, history: HistoryMessage[]): HistorySliceDecision;
63
+ export declare function decideCachedHistorySlice(savedCount: number, history: HistoryMessage[]): HistorySliceDecision;
57
64
  export {};
@@ -30,29 +30,31 @@ export function shouldCommitCachedTurn(context) {
30
30
  * attachment probing). Kept here so the decision can be unit-tested in
31
31
  * isolation.
32
32
  *
33
- * The key regression guard: when a non-zero `savedCount` would slice
34
- * the history down to an empty array, it is treated as stale — the
35
- * caller falls back to sending the system-stripped full history rather
33
+ * A non-zero `savedCount` is itself the proof that a cache holds a rendered
34
+ * prefix: it is recorded only after a save is verified. It means the prefix
35
+ * covers that many messages, so slicing from it drops exactly what the cache
36
+ * supplies — the system message among them. Without a usable boundary the
37
+ * cache supplies nothing, so the whole history goes to the addon, system
38
+ * message included: anything held back would simply be missing from the
39
+ * prompt.
40
+ *
41
+ * The regression guard: a non-zero `savedCount` that slices the history down
42
+ * to an empty array is stale, and the caller resends the full history rather
36
43
  * than handing the model an empty payload.
37
44
  */
38
- export function decideCachedHistorySlice(savedCount, cacheExists, history) {
39
- if (!cacheExists || history.length === 0) {
40
- return {
41
- messages: history.filter((msg) => msg.role !== 'system'),
42
- clearStaleCount: false
43
- };
44
- }
45
- const canSlice = savedCount > 0 && savedCount <= history.length;
46
- const sliced = canSlice ? history.slice(savedCount) : null;
45
+ export function decideCachedHistorySlice(savedCount, history) {
46
+ const hasCachedPrefix = savedCount > 0 && history.length > 0;
47
+ const sliced = hasCachedPrefix && savedCount <= history.length ? history.slice(savedCount) : null;
47
48
  // A non-null slice that is empty means the saved count is stale: the
48
49
  // cached turn boundary is claiming the entire current history is
49
50
  // already cached, which happens when a previous turn was cancelled
50
51
  // mid-decode and still recorded `history.length + 1`. Treat it as a
51
- // bad state and resend the full (system-stripped) history.
52
+ // bad state and resend the full history.
52
53
  const useSlice = sliced !== null && sliced.length > 0;
53
- const messages = useSlice ? sliced : history.filter((msg) => msg.role !== 'system');
54
54
  return {
55
- messages,
56
- clearStaleCount: !useSlice && savedCount > 0
55
+ messages: useSlice ? sliced : history,
56
+ // Only a boundary that was actually consulted can be stale. Absent a
57
+ // history to slice, the count is simply unused and left alone.
58
+ clearStaleCount: hasCachedPrefix && !useSlice
57
59
  };
58
60
  }