@qvac/inference 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/NOTICE +95 -39
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/delete-cache.d.ts +7 -1
  5. package/dist/api/delete-cache.js +11 -1
  6. package/dist/api/index.d.ts +2 -1
  7. package/dist/api/index.js +2 -1
  8. package/dist/api/text-to-speech.d.ts +31 -4
  9. package/dist/api/text-to-speech.js +117 -21
  10. package/dist/api/vector-index.d.ts +142 -0
  11. package/dist/api/vector-index.js +168 -0
  12. package/dist/api/world.d.ts +2 -2
  13. package/dist/errors/index.d.ts +12 -0
  14. package/dist/errors/index.js +21 -0
  15. package/dist/handlers/delete-cache.js +3 -0
  16. package/dist/handlers/vector-index.d.ts +2 -0
  17. package/dist/handlers/vector-index.js +40 -0
  18. package/dist/index.d.ts +1 -1
  19. package/dist/index.js +1 -1
  20. package/dist/models/registry/models.d.ts +351 -12
  21. package/dist/models/registry/models.js +2271 -1957
  22. package/dist/models/registry/resource-profiles.js +135 -0
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  30. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  31. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  32. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  34. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  36. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  37. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  38. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  39. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  40. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  41. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  44. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  45. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  46. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  47. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
  49. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
  50. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  51. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  52. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
  53. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  54. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  55. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  56. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  57. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  58. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  59. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  60. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  61. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  62. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  63. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  64. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  65. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  66. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  67. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  68. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  70. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  71. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  72. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  73. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  74. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  75. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  76. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  77. package/dist/plugins/index.d.ts +1 -1
  78. package/dist/plugins/index.js +1 -1
  79. package/dist/plugins/ops/load-model.js +22 -2
  80. package/dist/plugins/ops/translate.js +58 -12
  81. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  82. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  83. package/dist/plugins/registry.d.ts +15 -0
  84. package/dist/plugins/registry.js +133 -15
  85. package/dist/plugins/turbovec-backend.d.ts +17 -0
  86. package/dist/plugins/turbovec-backend.js +99 -0
  87. package/dist/registry.js +4 -0
  88. package/dist/resources/model-fit/assess.js +169 -68
  89. package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
  90. package/dist/resources/model-fit/estimators/llm.js +3 -1
  91. package/dist/resources/model-fit/floor.d.ts +31 -0
  92. package/dist/resources/model-fit/floor.js +65 -0
  93. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  94. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  95. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  96. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  97. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  98. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  99. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  100. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  101. package/dist/resources/native.js +5 -4
  102. package/dist/resources/types.d.ts +1 -3
  103. package/dist/runtime/env.d.ts +2 -0
  104. package/dist/runtime/env.js +9 -2
  105. package/dist/runtime/lifecycle.js +2 -0
  106. package/dist/runtime/model-registry.d.ts +3 -1
  107. package/dist/runtime/model-registry.js +2 -1
  108. package/dist/runtime/request-context.js +17 -0
  109. package/dist/runtime/vector-index-backend.d.ts +25 -0
  110. package/dist/runtime/vector-index-backend.js +1 -0
  111. package/dist/runtime/vector-index-registry.d.ts +76 -0
  112. package/dist/runtime/vector-index-registry.js +161 -0
  113. package/dist/schemas/assess-model-fit.d.ts +84 -0
  114. package/dist/schemas/assess-model-fit.js +72 -3
  115. package/dist/schemas/audio-gen.d.ts +482 -7
  116. package/dist/schemas/audio-gen.js +376 -17
  117. package/dist/schemas/batch-completion-stream.d.ts +10 -0
  118. package/dist/schemas/batch-completion-stream.js +5 -3
  119. package/dist/schemas/bci.d.ts +1 -1
  120. package/dist/schemas/common.d.ts +782 -19
  121. package/dist/schemas/common.js +8 -1
  122. package/dist/schemas/completion-event.d.ts +14 -1
  123. package/dist/schemas/completion-event.js +4 -0
  124. package/dist/schemas/completion-stream.d.ts +22 -0
  125. package/dist/schemas/completion-stream.js +41 -4
  126. package/dist/schemas/config.d.ts +48 -30
  127. package/dist/schemas/errors.d.ts +4 -0
  128. package/dist/schemas/errors.js +22 -0
  129. package/dist/schemas/index.d.ts +1 -0
  130. package/dist/schemas/index.js +1 -0
  131. package/dist/schemas/llamacpp-config.d.ts +4 -8
  132. package/dist/schemas/llamacpp-config.js +4 -4
  133. package/dist/schemas/load-model.d.ts +2604 -180
  134. package/dist/schemas/model-config-schemas.d.ts +228 -5
  135. package/dist/schemas/model-ops.d.ts +3 -0
  136. package/dist/schemas/model-ops.js +6 -0
  137. package/dist/schemas/plugin.d.ts +21 -11
  138. package/dist/schemas/sdcpp-config.d.ts +221 -4
  139. package/dist/schemas/sdcpp-config.js +128 -22
  140. package/dist/schemas/text-to-speech.d.ts +780 -19
  141. package/dist/schemas/text-to-speech.js +298 -44
  142. package/dist/schemas/transcription-config.js +7 -4
  143. package/dist/schemas/vector-index.d.ts +194 -0
  144. package/dist/schemas/vector-index.js +185 -0
  145. package/dist/utils/addon-responses.d.ts +30 -3
  146. package/dist/utils/aggregate-events.d.ts +3 -1
  147. package/dist/utils/aggregate-events.js +7 -1
  148. package/dist/utils/tts-stats.d.ts +15 -11
  149. package/dist/utils/tts-stats.js +33 -11
  150. package/package.json +29 -26
@@ -6,27 +6,130 @@ import { isMobile } from '../../../../runtime/state.js';
6
6
  import { getKVCacheDir } from '../../../../utils/index.js';
7
7
  import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
8
8
  import { getEngineLogger } from '../../../../logging/index.js';
9
+ import { z } from 'zod';
9
10
  // Used by cross-model paths that have no `RequestContext` (e.g.
10
11
  // `deleteKvCacheState`). Per-session call sites receive a logger from
11
12
  // the caller — typically `withRequestContext(...)`.
12
13
  const moduleLogger = getEngineLogger();
14
+ /**
15
+ * Coordinates six KV-cache state layers:
16
+ *
17
+ * 1. `cachedPrefixes` — saved message boundaries, and whether a tool
18
+ * block was rendered into them.
19
+ * 2. `initializedCaches` — caches this process has established.
20
+ * 3. On-disk `.bin` files written by the addon.
21
+ * 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
22
+ * boundary outlives the process; see `readPrefixSidecar`.
23
+ * 5. `activeCachePaths` — per-path refs that block in-flight eviction.
24
+ * 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
25
+ *
26
+ * Every turn must finish through `commitTurn`, `rollback`, or the
27
+ * non-destructive `releaseTurn` so all inference state stays aligned,
28
+ * the active-path ref is released, and marker metadata follows the
29
+ * cache directory lifecycle.
30
+ */
31
+ // ----- module-scoped state. The session is the single mutation point
32
+ // for the in-memory KV-cache bookkeeping. -----
33
+ /** What the kv-cache file at a given path is known to hold. */
34
+ const cachedPrefixSchema = z.object({
35
+ /** Number of chat messages the file on disk is known to cover. */
36
+ messages: z.number().int().nonnegative(),
37
+ /**
38
+ * Whether a static tool block was rendered into that prefix. Tracked
39
+ * rather than inferred from `messages`, because a committed turn is not
40
+ * proof that its tool block reached the model: the addon drops tools and
41
+ * still returns a usable prompt when the chat template rejects them.
42
+ */
43
+ toolBlock: z.boolean()
44
+ });
45
+ // The sidecar is written after the addon has already saved the `.bin`, so a
46
+ // crash between the two leaves a boundary that describes an older file. The
47
+ // `.bin` size is a function of the tokens it holds: it is unchanged when the
48
+ // addon re-saves the same state (cancel rewind, key switch) and larger once
49
+ // another turn landed in it, which is the case the fingerprint must catch.
50
+ const persistedPrefixSchema = cachedPrefixSchema.extend({
51
+ binSize: z.number().int().nonnegative()
52
+ });
13
53
  /**
14
54
  * What the kv-cache file on disk is known to cover, keyed by cache path.
15
- * Written by `commitTurn`, read by `getSavedCount`, deleted by `rollback` /
16
- * `delete` / `dropStaleSavedCount`. The same INVARIANT that existed in
17
- * `kv-cache-state.ts` still holds: an entry is present only when the
18
- * corresponding `.bin` file is considered trustworthy. Cancelled or
19
- * zero-token turns must remove the entry so the next-turn slice doesn't read
20
- * a stale boundary.
55
+ * Written by `commitTurn`, deleted by `rollback`, `delete`,
56
+ * `dropStaleSavedCount` and the orphan clear in `beginTurn`. The same
57
+ * INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
58
+ * present only when the corresponding `.bin` file is considered trustworthy,
59
+ * so any turn that leaves the file holding something the boundary does not
60
+ * describe has to remove it.
61
+ *
62
+ * This map outranks the sidecar. Both are written by the same commit, so an
63
+ * entry here is never the older copy, and `restorePersistedCache` reads the
64
+ * file only to repopulate an empty map after a restart.
21
65
  */
22
66
  const cachedPrefixes = new Map();
67
+ const PREFIX_SIDECAR_SUFFIX = '.meta.json';
68
+ function prefixSidecarPath(cachePath) {
69
+ return `${cachePath}${PREFIX_SIDECAR_SUFFIX}`;
70
+ }
71
+ async function writePrefixSidecar(cachePath, prefix, logger) {
72
+ try {
73
+ const { size } = await fsPromises.stat(cachePath);
74
+ await fsPromises.writeFile(prefixSidecarPath(cachePath), JSON.stringify({ ...prefix, binSize: size }));
75
+ }
76
+ catch (error) {
77
+ // Only the file goes: a sidecar left from the previous commit describes a
78
+ // shorter `.bin` than the one now on disk. The in-memory boundary is the
79
+ // one this process keeps slicing against and stays.
80
+ await removePrefixSidecar(cachePath);
81
+ logger.warn(`[kv-cache] Failed to persist saved-message boundary; this process stays warm but a restart will start the cache cold. path=${cachePath} error=${error instanceof Error ? error.message : String(error)}`);
82
+ }
83
+ }
84
+ // Anything unreadable, malformed, or describing a `.bin` of a different size
85
+ // is discarded: the cache is then used from a cold boundary, which is the
86
+ // pre-sidecar behaviour. Only ever consulted for a path `cachedPrefixes` has
87
+ // no entry for, so it can never contradict a live boundary. Never throws: the
88
+ // caller holds the cache-path write lock and a `.bin` can vanish under it
89
+ // (`deleteKvCacheState` takes no locks).
90
+ async function readPrefixSidecar(cachePath) {
91
+ let parsed;
92
+ let binSize;
93
+ try {
94
+ const raw = await fsPromises.readFile(prefixSidecarPath(cachePath), 'utf8');
95
+ const result = persistedPrefixSchema.safeParse(JSON.parse(raw));
96
+ if (!result.success)
97
+ return null;
98
+ parsed = result.data;
99
+ binSize = (await fsPromises.stat(cachePath)).size;
100
+ }
101
+ catch {
102
+ return null;
103
+ }
104
+ if (binSize !== parsed.binSize) {
105
+ // The `.bin` changed after this boundary was recorded, so the file is
106
+ // stale whatever wrote it. Only the file: the caller guarantees the map
107
+ // holds nothing for this path.
108
+ await removePrefixSidecar(cachePath);
109
+ return null;
110
+ }
111
+ return { messages: parsed.messages, toolBlock: parsed.toolBlock };
112
+ }
113
+ /** Drops the persisted boundary, leaving whatever is in memory alone. */
114
+ async function removePrefixSidecar(cachePath) {
115
+ try {
116
+ await fsPromises.unlink(prefixSidecarPath(cachePath));
117
+ }
118
+ catch {
119
+ // No sidecar for this path.
120
+ }
121
+ }
122
+ /** Drops the boundary everywhere: the cache it describes is gone or untrusted. */
123
+ async function forgetPrefix(cachePath) {
124
+ cachedPrefixes.delete(cachePath);
125
+ await removePrefixSidecar(cachePath);
126
+ }
23
127
  /**
24
- * In-memory registry of caches initialized this session. The addon
25
- * defers disk writes, so the absence of a `.bin` file on disk isn't
26
- * proof that the cache hasn't been primed in this process. Keyed
27
- * by the resolved cache path, so aliased keys that name one file share an
28
- * entry and on-disk caches from older process runs still hit the lazy-load
29
- * path in `beginTurn`.
128
+ * In-memory registry of caches established this session, recorded once a save
129
+ * is verified. The addon defers disk writes, so the absence of a `.bin` file on
130
+ * disk isn't proof that no cache exists for this process. Keyed by the resolved
131
+ * cache path, so aliased keys that name one file share an entry and on-disk
132
+ * caches from older process runs still hit the lazy-load path in `beginTurn`.
30
133
  */
31
134
  const initializedCaches = new Set();
32
135
  const activeCachePaths = new Map();
@@ -128,6 +231,25 @@ function isCacheKeyActive(cacheKey) {
128
231
  function getAutoCacheMaxBytes() {
129
232
  return isMobile() ? MOBILE_AUTO_CACHE_MAX_BYTES : DESKTOP_AUTO_CACHE_MAX_BYTES;
130
233
  }
234
+ /**
235
+ * Deletes planned auto-cache keys and reports how many went.
236
+ *
237
+ * Planning and eviction are separate steps, so a turn can take a key in
238
+ * between: each one is re-checked against the active paths under the
239
+ * cache-state lock rather than trusted from the plan.
240
+ */
241
+ async function evictPlannedAutoCaches(cacheKeys) {
242
+ let evicted = 0;
243
+ await withCacheStateLock(async () => {
244
+ for (const cacheKey of cacheKeys) {
245
+ if (isCacheKeyActive(cacheKey))
246
+ continue;
247
+ await deleteKvCacheState({ kvCacheKey: cacheKey });
248
+ evicted++;
249
+ }
250
+ });
251
+ return evicted;
252
+ }
131
253
  async function withCacheStateLock(operation) {
132
254
  const previous = cacheStateLockTail;
133
255
  let releaseLock = () => { };
@@ -156,20 +278,12 @@ async function maybeSweepAutoCaches(logger, overrides) {
156
278
  const sweep = async () => {
157
279
  try {
158
280
  const cacheKeys = await planAutoCacheEvictions({
159
- activeCachePaths: Array.from(activeCachePaths.keys()),
281
+ activeCachePaths: snapshotActivePaths(),
160
282
  maxBytes: overrides?.maxBytes ?? getAutoCacheMaxBytes(),
161
283
  maxIdleMs: overrides?.maxIdleMs ?? AUTO_CACHE_MAX_IDLE_MS,
162
284
  nowMs
163
285
  });
164
- let evictionCount = 0;
165
- await withCacheStateLock(async () => {
166
- for (const cacheKey of cacheKeys) {
167
- if (isCacheKeyActive(cacheKey))
168
- continue;
169
- await deleteKvCacheState({ kvCacheKey: cacheKey });
170
- evictionCount++;
171
- }
172
- });
286
+ const evictionCount = await evictPlannedAutoCaches(cacheKeys);
173
287
  if (evictionCount > 0) {
174
288
  logger.debug(`[kv-cache] Evicted ${evictionCount} inactive auto-cache entries`);
175
289
  }
@@ -192,6 +306,30 @@ function scheduleAutoCacheSweep(logger) {
192
306
  logger.warn(`[kv-cache] Failed to schedule auto-cache retention sweep: ${error instanceof Error ? error.message : String(error)}`);
193
307
  });
194
308
  }
309
+ /**
310
+ * Drops every auto cache no turn is holding, whatever its age or size.
311
+ *
312
+ * The on-demand counterpart to `maybeSweepAutoCaches`, which applies the
313
+ * standing quota and TTL instead. Reusing the planner keeps the "is this an
314
+ * auto cache" decision in one place: only `.auto-cache-<key>` markers are
315
+ * considered, so caller-owned named caches stay out of scope.
316
+ */
317
+ async function deleteInactiveAutoCaches() {
318
+ const cacheKeys = await planAutoCacheEvictions({
319
+ activeCachePaths: snapshotActivePaths(),
320
+ // The quota pass stops at `retainedBytes <= maxBytes`, and a total reaches
321
+ // 0 while zero-byte caches remain (an empty `.bin` from an interrupted save).
322
+ // A negative quota is unreachable, so every inactive entry is selected.
323
+ maxBytes: -1,
324
+ // No age threshold: the quota pass above already selects everything.
325
+ maxIdleMs: 0,
326
+ nowMs: Date.now()
327
+ });
328
+ const evicted = await evictPlannedAutoCaches(cacheKeys);
329
+ if (evicted > 0) {
330
+ moduleLogger.debug(`[kv-cache] Reclaimed ${evicted} inactive auto-cache entries`);
331
+ }
332
+ }
195
333
  // ----- factory -----
196
334
  /**
197
335
  * Construct a session bound to one `(modelId, turn-owning request)`
@@ -219,7 +357,7 @@ export function createKvCacheSession(modelId, options) {
219
357
  releaseWriteLock,
220
358
  committed: false,
221
359
  rolledBack: false,
222
- freshlyPrimed: false
360
+ createdByThisTurn: false
223
361
  });
224
362
  markCachePathActive(cachePath);
225
363
  return handle;
@@ -241,7 +379,6 @@ export function createKvCacheSession(modelId, options) {
241
379
  }
242
380
  throw error;
243
381
  }
244
- // A turn cancelled by the time it holds the lock must not prime (native work).
245
382
  // getCacheFilePath already mkdir'd the parent, so prune it before surfacing
246
383
  // the cancellation the plugin rides.
247
384
  if (input.signal?.aborted) {
@@ -249,36 +386,25 @@ export function createKvCacheSession(modelId, options) {
249
386
  releaseWriteLock();
250
387
  throw new CacheLockAbortError(input.signal.reason);
251
388
  }
389
+ // In-memory registry check first — the addon defers disk writes, so a
390
+ // just-saved cache may not yet exist on disk. If the in-memory flag isn't
391
+ // set, fall back to a filesystem probe so caches surviving across process
392
+ // restarts still hit the reuse path. Resolved before the handle is made so
393
+ // it snapshots the restored boundary.
394
+ let exists = initializedCaches.has(cachePath);
395
+ if (!exists)
396
+ exists = await restorePersistedCache(cachePath);
397
+ // A boundary without its `.bin` describes a cache that no longer exists.
398
+ if (!exists)
399
+ await forgetPrefix(cachePath);
252
400
  const handle = makeHandle(cachePath, undefined, releaseWriteLock, input.signal);
253
401
  try {
254
- // In-memory registry check first — the addon defers disk writes, so
255
- // a freshly-primed cache may not yet exist on disk. If the
256
- // in-memory flag isn't set, fall back to a filesystem probe so
257
- // caches surviving across process restarts still hit the reuse path.
258
- let exists = initializedCaches.has(cachePath);
259
- if (!exists) {
260
- try {
261
- await fsPromises.access(cachePath);
262
- exists = true;
263
- initializedCaches.add(cachePath);
264
- }
265
- catch {
266
- exists = false;
267
- }
268
- }
269
402
  logCacheStatus(input.customKey, exists);
270
403
  if (!exists) {
271
- // Recreate the parent dir if a same-key peer's rollback pruned it after our lock wait.
404
+ // Recreate the parent dir if a same-key peer's rollback pruned it after
405
+ // our lock wait, so the addon's own save has somewhere to land.
272
406
  await fsPromises.mkdir(path.dirname(cachePath), { recursive: true });
273
- // The access probe / mkdir above yielded, so a cancel may have landed
274
- // since the acquire-time check — re-check right before native priming.
275
- // The catch below prunes the directory just created.
276
- if (input.signal?.aborted)
277
- throw new CacheLockAbortError(input.signal.reason);
278
- await input.primeIfMissing(cachePath);
279
- await verifyPrimedFile(cachePath, logger);
280
- initializedCaches.add(cachePath);
281
- turnState.get(handle).freshlyPrimed = true;
407
+ turnState.get(handle).createdByThisTurn = true;
282
408
  }
283
409
  return handle;
284
410
  }
@@ -316,7 +442,6 @@ export function createKvCacheSession(modelId, options) {
316
442
  }
317
443
  throw error;
318
444
  }
319
- // A turn cancelled by the time it holds the lock must not prime (native work).
320
445
  // Discovery already mkdir'd the parent and wrote the auto marker; clean both
321
446
  // before surfacing the cancellation.
322
447
  if (input.signal?.aborted) {
@@ -354,17 +479,10 @@ export function createKvCacheSession(modelId, options) {
354
479
  logCacheStatus('auto', cacheExists);
355
480
  try {
356
481
  if (!cacheExists) {
357
- // Recreate the parent dir if a same-file peer's rename pruned it.
482
+ // Recreate the parent dir if a same-file peer's rename pruned it, so
483
+ // the addon's own save has somewhere to land.
358
484
  await fsPromises.mkdir(path.dirname(cachePath), { recursive: true });
359
- // The discovery / mkdir above yielded, so a cancel may have landed since
360
- // the acquire-time check — re-check right before native priming. The
361
- // catch below prunes the directory and auto marker.
362
- if (input.signal?.aborted)
363
- throw new CacheLockAbortError(input.signal.reason);
364
- await input.primeIfMissing(cachePath);
365
- await verifyPrimedFile(cachePath, logger);
366
- initializedCaches.add(cachePath);
367
- turnState.get(handle).freshlyPrimed = true;
485
+ turnState.get(handle).createdByThisTurn = true;
368
486
  }
369
487
  return handle;
370
488
  }
@@ -397,13 +515,13 @@ export function createKvCacheSession(modelId, options) {
397
515
  // at the same path. Verify the file persisted (the addon
398
516
  // currently swallows save errors — see TODO in
399
517
  // `verifySaveAndRecord`) and record the new boundary.
400
- const ok = await verifySaveAndRecord(state.cachePath, result.messageCount, result.toolBlockCached);
518
+ const prefix = { messages: result.messageCount, toolBlock: result.toolBlockCached };
519
+ const ok = await verifySaveAndRecord(state.cachePath, prefix);
401
520
  if (!ok) {
402
- // The expected save didn't land — treat the turn as a rollback
403
- // so the next turn re-primes cleanly.
404
521
  await runRollback(state);
405
522
  return;
406
523
  }
524
+ await writePrefixSidecar(state.cachePath, prefix, logger);
407
525
  state.committed = true;
408
526
  releaseCachePath(state.cachePath);
409
527
  state.releaseWriteLock();
@@ -446,11 +564,11 @@ export function createKvCacheSession(modelId, options) {
446
564
  releaseCachePath(sourceCachePath);
447
565
  state.cachePath = result.targetCachePath;
448
566
  state.autoCacheKey = targetCacheKey;
567
+ await forgetPrefix(sourceCachePath);
449
568
  await pruneEmptyCacheDirectories(sourceCachePath, snapshotActivePaths());
450
569
  if (sourceCacheKey !== undefined) {
451
570
  await removeAutoCacheMarkerIfMissing(sourceCacheKey);
452
571
  }
453
- cachedPrefixes.delete(sourceCachePath);
454
572
  // state.cachePath was just reassigned to the target; clear the SOURCE
455
573
  // entry, not the freshly-valid target.
456
574
  initializedCaches.delete(sourceCachePath);
@@ -461,7 +579,10 @@ export function createKvCacheSession(modelId, options) {
461
579
  await runRollback(state);
462
580
  return;
463
581
  }
464
- const ok = await verifySaveAndRecord(result.targetCachePath, result.messageCount, result.toolBlockCached);
582
+ const ok = await verifySaveAndRecord(result.targetCachePath, {
583
+ messages: result.messageCount,
584
+ toolBlock: result.toolBlockCached
585
+ });
465
586
  if (!ok) {
466
587
  // Rename succeeded but the file isn't where we expected. Roll back via
467
588
  // the target path instead of the (now-empty) source.
@@ -494,9 +615,8 @@ export function createKvCacheSession(modelId, options) {
494
615
  return;
495
616
  if (state.committed || state.rolledBack)
496
617
  return;
497
- // A cache this same turn primed has no committed state to keep — a failed
498
- // first turn must not leave its own prime behind.
499
- if (state.freshlyPrimed) {
618
+ // Nothing committed exists to keep, so a failed first turn is destructive.
619
+ if (state.createdByThisTurn) {
500
620
  await runRollback(state);
501
621
  return;
502
622
  }
@@ -515,8 +635,13 @@ export function createKvCacheSession(modelId, options) {
515
635
  await fsPromises.unlink(state.cachePath);
516
636
  }
517
637
  catch (unlinkError) {
518
- logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
638
+ // A turn that fails before the addon's save has no file to remove, so
639
+ // ENOENT leaves no stale state behind and is not worth a warning.
640
+ if (unlinkError.code !== 'ENOENT') {
641
+ logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
642
+ }
519
643
  }
644
+ await forgetPrefix(state.cachePath);
520
645
  // Release before pruning so an empty parent can go; a sibling still holding
521
646
  // the path keeps it in the active snapshot and protects the directory.
522
647
  releaseCachePath(state.cachePath);
@@ -525,17 +650,16 @@ export function createKvCacheSession(modelId, options) {
525
650
  await removeAutoCacheMarkerIfMissing(state.autoCacheKey);
526
651
  }
527
652
  initializedCaches.delete(state.cachePath);
528
- cachedPrefixes.delete(state.cachePath);
529
653
  state.rolledBack = true;
530
654
  state.releaseWriteLock();
531
655
  if (state.autoCacheKey !== undefined)
532
656
  scheduleAutoCacheSweep(logger);
533
657
  }
534
- function dropStaleSavedCount(turn) {
658
+ async function dropStaleSavedCount(turn) {
535
659
  const state = turnState.get(turn);
536
660
  if (!state)
537
661
  return;
538
- cachedPrefixes.delete(state.cachePath);
662
+ await forgetPrefix(state.cachePath);
539
663
  }
540
664
  return {
541
665
  beginTurn,
@@ -548,9 +672,10 @@ export function createKvCacheSession(modelId, options) {
548
672
  // ----- module-level administrative API -----
549
673
  /**
550
674
  * Atomically delete every layer of KV-cache state for a
551
- * `(kvCacheKey, modelId)` pair, or wipe everything. Single entry point
552
- * — the only mutation point for cross-model state outside of
553
- * turn-scoped `commitTurn`/`rollback`.
675
+ * `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
676
+ * caches no turn is holding. Single entry point — the only mutation
677
+ * point for cross-model state outside of turn-scoped
678
+ * `commitTurn`/`rollback`.
554
679
  *
555
680
  * Why this isn't a method on `KvCacheSession`: deletes are
556
681
  * cross-model (`all: true` has no model; the keyed form has
@@ -575,10 +700,14 @@ export function createKvCacheSession(modelId, options) {
575
700
  * cleared without a lock, so an interleaving that lands the delete's
576
701
  * in-memory cleanup after a concurrent turn has already renamed/committed
577
702
  * its file splits state: the file stays on disk while its saved-count and
578
- * init flag are cleared, so the next turn sees the file, skips priming, and
579
- * reports `savedCount=0`. Callers must not delete a key that is in active use.
703
+ * init flag are cleared, so the next turn loads the file and reports
704
+ * `savedCount=0`. Callers must not delete a key that is in active use.
705
+ * The `auto` target is the exception: it skips keys an in-flight turn holds
706
+ * and clears each one under the cache-state lock.
580
707
  */
581
708
  export async function deleteKvCacheState(target) {
709
+ if ('auto' in target)
710
+ return deleteInactiveAutoCaches();
582
711
  if ('all' in target) {
583
712
  const removed = await deleteCacheUtil({ all: true });
584
713
  cachedPrefixes.clear();
@@ -609,62 +738,6 @@ export async function deleteKvCacheState(target) {
609
738
  clearInitializedCachesByPrefix(removedPath, path.sep);
610
739
  }
611
740
  // ----- private helpers -----
612
- /**
613
- * Verify that the addon actually persisted a usable cache file after a
614
- * prime. Mirrors the `verifySaveAndRecord` access-probe used at commit
615
- * time, applied at prime time so the session doesn't mark a cache
616
- * `initializedCaches.add(...)` against a path that's missing or empty
617
- * on disk.
618
- *
619
- * Failure modes this catches:
620
- *
621
- * - The addon's `model.run({ saveSessionPath })` was interrupted
622
- * before the save call ran (e.g. signal abort during prefill); the
623
- * prime closure resolves cleanly because addon save errors are not
624
- * propagated, but no file is on disk.
625
- * - The addon's `llama_state_save_file` was called but produced an
626
- * empty file (out-of-space / fs error swallowed by the addon).
627
- *
628
- * Failure modes this does **NOT** catch:
629
- *
630
- * - A partial-but-nonzero file written by the addon (e.g. header +
631
- * truncated KV state). Catching this requires either an
632
- * addon-side change (have `CacheManager::writeCacheFile` check the
633
- * return value of `llama_state_save_file` and throw on failure) or
634
- * a structural hash check we can't currently compute
635
- * engine-side. Filed as a follow-up — see `cache-api.md` in the addon
636
- * repo / tracking ticket.
637
- *
638
- * On failure we best-effort `unlink` an empty leftover file (so the
639
- * next existence probe doesn't trust it) and throw — the handler in
640
- * `completion-stream.ts` lets the error propagate up and no
641
- * `initializedCaches` entry is recorded.
642
- */
643
- async function verifyPrimedFile(cachePath, logger) {
644
- let stats;
645
- try {
646
- stats = await fsPromises.stat(cachePath);
647
- }
648
- catch (statError) {
649
- // ENOENT is the common case here — addon prime returned without
650
- // calling save (most often: signal abort during prefill).
651
- await pruneEmptyCacheDirectories(cachePath, snapshotActivePaths());
652
- throw new Error(`[kv-cache] prime closure resolved but no cache file was written. path=${cachePath} cause=${statError instanceof Error ? statError.message : String(statError)}`);
653
- }
654
- if (stats.size === 0) {
655
- // Best-effort cleanup so a future probe doesn't trust the empty
656
- // file. Unlink failure is non-fatal — we still throw on the
657
- // primary "prime didn't persist" condition.
658
- try {
659
- await fsPromises.unlink(cachePath);
660
- await pruneEmptyCacheDirectories(cachePath, snapshotActivePaths());
661
- }
662
- catch (unlinkError) {
663
- logger.warn(`[kv-cache] Failed to remove empty primed cache file. path=${cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
664
- }
665
- throw new Error(`[kv-cache] prime closure resolved but cache file is empty. path=${cachePath}`);
666
- }
667
- }
668
741
  /**
669
742
  * Verify the addon actually persisted the cache file before recording
670
743
  * its message count. The addon currently swallows write errors
@@ -676,10 +749,11 @@ async function verifyPrimedFile(cachePath, logger) {
676
749
  * false), drop the `access()` probe and wrap the `model.run()` call in
677
750
  * a real try/catch that forwards the error.
678
751
  */
679
- async function verifySaveAndRecord(cachePath, messageCount, toolBlockCached) {
752
+ async function verifySaveAndRecord(cachePath, prefix) {
680
753
  try {
681
754
  await fsPromises.access(cachePath);
682
- cachedPrefixes.set(cachePath, { messages: messageCount, toolBlock: toolBlockCached });
755
+ cachedPrefixes.set(cachePath, prefix);
756
+ initializedCaches.add(cachePath);
683
757
  return true;
684
758
  }
685
759
  catch (err) {
@@ -688,6 +762,30 @@ async function verifySaveAndRecord(cachePath, messageCount, toolBlockCached) {
688
762
  return false;
689
763
  }
690
764
  }
765
+ /**
766
+ * Adopt a `.bin` left by an earlier process run: record it as established and
767
+ * pick up the boundary committed alongside it. Returns whether the file exists.
768
+ *
769
+ * An auto-rename commits its target into `cachedPrefixes` without an init
770
+ * flag, so this can run for a path the map already knows. The map wins there
771
+ * and the sidecar is left unread, which keeps the two from ever having to be
772
+ * reconciled.
773
+ */
774
+ async function restorePersistedCache(cachePath) {
775
+ try {
776
+ await fsPromises.access(cachePath);
777
+ }
778
+ catch {
779
+ return false;
780
+ }
781
+ initializedCaches.add(cachePath);
782
+ if (cachedPrefixes.has(cachePath))
783
+ return true;
784
+ const prefix = await readPrefixSidecar(cachePath);
785
+ if (prefix !== null)
786
+ cachedPrefixes.set(cachePath, prefix);
787
+ return true;
788
+ }
691
789
  function clearCachedMessageCountsByPrefix(prefix, sep) {
692
790
  if (!prefix) {
693
791
  cachedPrefixes.clear();
@@ -733,6 +831,12 @@ export const __kvCacheSessionTestHooks = {
733
831
  getToolBlockCachedForTest(cachePath) {
734
832
  return cachedPrefixes.get(cachePath)?.toolBlock ?? false;
735
833
  },
834
+ getPrefixSidecarPathForTest(cachePath) {
835
+ return prefixSidecarPath(cachePath);
836
+ },
837
+ readPrefixSidecarForTest(cachePath) {
838
+ return readPrefixSidecar(cachePath);
839
+ },
736
840
  hasInitializedPath(cachePath) {
737
841
  return initializedCaches.has(cachePath);
738
842
  },
@@ -48,10 +48,17 @@ export declare function shouldCommitCachedTurn(context: CacheCommitContext): boo
48
48
  * attachment probing). Kept here so the decision can be unit-tested in
49
49
  * isolation.
50
50
  *
51
- * The key regression guard: when a non-zero `savedCount` would slice
52
- * the history down to an empty array, it is treated as stale — the
53
- * caller falls back to sending the system-stripped full history rather
51
+ * A non-zero `savedCount` is itself the proof that a cache holds a rendered
52
+ * prefix: it is recorded only after a save is verified. It means the prefix
53
+ * covers that many messages, so slicing from it drops exactly what the cache
54
+ * supplies — the system message among them. Without a usable boundary the
55
+ * cache supplies nothing, so the whole history goes to the addon, system
56
+ * message included: anything held back would simply be missing from the
57
+ * prompt.
58
+ *
59
+ * The regression guard: a non-zero `savedCount` that slices the history down
60
+ * to an empty array is stale, and the caller resends the full history rather
54
61
  * than handing the model an empty payload.
55
62
  */
56
- export declare function decideCachedHistorySlice(savedCount: number, cacheExists: boolean, history: HistoryMessage[]): HistorySliceDecision;
63
+ export declare function decideCachedHistorySlice(savedCount: number, history: HistoryMessage[]): HistorySliceDecision;
57
64
  export {};
@@ -30,29 +30,31 @@ export function shouldCommitCachedTurn(context) {
30
30
  * attachment probing). Kept here so the decision can be unit-tested in
31
31
  * isolation.
32
32
  *
33
- * The key regression guard: when a non-zero `savedCount` would slice
34
- * the history down to an empty array, it is treated as stale — the
35
- * caller falls back to sending the system-stripped full history rather
33
+ * A non-zero `savedCount` is itself the proof that a cache holds a rendered
34
+ * prefix: it is recorded only after a save is verified. It means the prefix
35
+ * covers that many messages, so slicing from it drops exactly what the cache
36
+ * supplies — the system message among them. Without a usable boundary the
37
+ * cache supplies nothing, so the whole history goes to the addon, system
38
+ * message included: anything held back would simply be missing from the
39
+ * prompt.
40
+ *
41
+ * The regression guard: a non-zero `savedCount` that slices the history down
42
+ * to an empty array is stale, and the caller resends the full history rather
36
43
  * than handing the model an empty payload.
37
44
  */
38
- export function decideCachedHistorySlice(savedCount, cacheExists, history) {
39
- if (!cacheExists || history.length === 0) {
40
- return {
41
- messages: history.filter((msg) => msg.role !== 'system'),
42
- clearStaleCount: false
43
- };
44
- }
45
- const canSlice = savedCount > 0 && savedCount <= history.length;
46
- const sliced = canSlice ? history.slice(savedCount) : null;
45
+ export function decideCachedHistorySlice(savedCount, history) {
46
+ const hasCachedPrefix = savedCount > 0 && history.length > 0;
47
+ const sliced = hasCachedPrefix && savedCount <= history.length ? history.slice(savedCount) : null;
47
48
  // A non-null slice that is empty means the saved count is stale: the
48
49
  // cached turn boundary is claiming the entire current history is
49
50
  // already cached, which happens when a previous turn was cancelled
50
51
  // mid-decode and still recorded `history.length + 1`. Treat it as a
51
- // bad state and resend the full (system-stripped) history.
52
+ // bad state and resend the full history.
52
53
  const useSlice = sliced !== null && sliced.length > 0;
53
- const messages = useSlice ? sliced : history.filter((msg) => msg.role !== 'system');
54
54
  return {
55
- messages,
56
- clearStaleCount: !useSlice && savedCount > 0
55
+ messages: useSlice ? sliced : history,
56
+ // Only a boundary that was actually consulted can be stale. Absent a
57
+ // history to slice, the count is simply unused and left alone.
58
+ clearStaleCount: hasCachedPrefix && !useSlice
57
59
  };
58
60
  }