@qvac/inference 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/NOTICE +50 -25
  2. package/README.md +44 -1
  3. package/dist/api/assess-model-fit.d.ts +4 -4
  4. package/dist/api/assess-model-fit.js +4 -4
  5. package/dist/api/audio-gen.d.ts +4 -1
  6. package/dist/api/audio-gen.js +4 -1
  7. package/dist/api/batch-completion.js +1 -1
  8. package/dist/api/completion-stream.js +1 -1
  9. package/dist/api/decide.d.ts +26 -0
  10. package/dist/api/decide.js +26 -0
  11. package/dist/api/get-model-info.d.ts +1 -1
  12. package/dist/api/index.d.ts +1 -0
  13. package/dist/api/index.js +1 -0
  14. package/dist/api/text-to-speech.d.ts +1 -0
  15. package/dist/api/text-to-speech.js +15 -19
  16. package/dist/api/transcribe.js +5 -2
  17. package/dist/api/video-request.d.ts +5 -5
  18. package/dist/api/video.d.ts +5 -4
  19. package/dist/api/video.js +5 -4
  20. package/dist/errors/index.d.ts +3 -0
  21. package/dist/errors/index.js +5 -0
  22. package/dist/handlers/assess-model-fit.d.ts +5 -3
  23. package/dist/handlers/assess-model-fit.js +80 -28
  24. package/dist/handlers/get-loaded-model-info.js +1 -2
  25. package/dist/handlers/load-model/safe-fetch.js +5 -6
  26. package/dist/handlers/registry.d.ts +3 -1
  27. package/dist/handlers/registry.js +3 -3
  28. package/dist/index.d.ts +1 -1
  29. package/dist/index.js +1 -1
  30. package/dist/logging/logger.js +1 -1
  31. package/dist/logging/utils.js +1 -1
  32. package/dist/models/registry/models.d.ts +563 -816
  33. package/dist/models/registry/models.js +2873 -3110
  34. package/dist/models/registry/resource-profiles.d.ts +1 -2
  35. package/dist/models/registry/resource-profiles.js +203 -2
  36. package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
  37. package/dist/plugins/builtin/asr-ggml/config.js +6 -0
  38. package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
  39. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
  40. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
  41. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
  42. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
  43. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
  44. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
  45. package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
  46. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
  47. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
  48. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
  49. package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
  50. package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
  51. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
  52. package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
  53. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
  54. package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
  55. package/dist/plugins/builtin/index.d.ts +2 -0
  56. package/dist/plugins/builtin/index.js +2 -0
  57. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
  58. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
  59. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
  60. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
  61. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
  62. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
  63. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
  64. package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
  65. package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
  66. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
  67. package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
  68. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
  69. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
  70. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
  71. package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
  72. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
  73. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
  74. package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
  75. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
  76. package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
  77. package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
  78. package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
  79. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
  80. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
  81. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
  82. package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
  83. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
  84. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
  85. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
  86. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
  87. package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
  88. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
  89. package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
  90. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
  91. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
  92. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
  93. package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
  94. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
  95. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
  96. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
  97. package/dist/plugins/ops/kv-cache-utils.js +10 -25
  98. package/dist/plugins/ops/load-model.js +21 -16
  99. package/dist/plugins/ops/transcribe.js +17 -1
  100. package/dist/registry.js +1 -0
  101. package/dist/resources/model-fit/assess.d.ts +15 -23
  102. package/dist/resources/model-fit/assess.js +207 -271
  103. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
  104. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
  105. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
  106. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
  107. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
  108. package/dist/resources/model-fit/floor.d.ts +10 -10
  109. package/dist/resources/model-fit/floor.js +13 -13
  110. package/dist/resources/model-fit/kv-cache.d.ts +23 -0
  111. package/dist/resources/model-fit/kv-cache.js +85 -0
  112. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
  113. package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
  114. package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
  115. package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
  116. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
  117. package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
  118. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
  119. package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
  120. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
  121. package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
  122. package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
  123. package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
  124. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
  125. package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
  126. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
  127. package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
  128. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
  129. package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
  130. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
  131. package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
  132. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
  133. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
  134. package/dist/resources/model-fit/platform.d.ts +4 -4
  135. package/dist/resources/model-fit/platform.js +6 -8
  136. package/dist/resources/model-fit/types.d.ts +3 -117
  137. package/dist/rpc/inventory.js +5 -7
  138. package/dist/runtime/env.d.ts +0 -2
  139. package/dist/runtime/env.js +2 -9
  140. package/dist/runtime/model-config-utils.js +6 -1
  141. package/dist/runtime/request-context.d.ts +1 -1
  142. package/dist/runtime/request-context.js +8 -0
  143. package/dist/runtime/state.js +5 -0
  144. package/dist/schemas/assess-model-fit.d.ts +100 -43
  145. package/dist/schemas/assess-model-fit.js +66 -29
  146. package/dist/schemas/audio-gen.d.ts +186 -18
  147. package/dist/schemas/audio-gen.js +180 -48
  148. package/dist/schemas/batch-completion-stream.d.ts +202 -34
  149. package/dist/schemas/bci-config.d.ts +2 -0
  150. package/dist/schemas/bci.d.ts +23 -2
  151. package/dist/schemas/cancel.d.ts +5 -0
  152. package/dist/schemas/cancel.js +1 -0
  153. package/dist/schemas/common.d.ts +1083 -158
  154. package/dist/schemas/common.js +5 -2
  155. package/dist/schemas/completion-stream.d.ts +232 -37
  156. package/dist/schemas/completion-stream.js +64 -20
  157. package/dist/schemas/config.d.ts +477 -12
  158. package/dist/schemas/config.js +21 -1
  159. package/dist/schemas/decision.d.ts +602 -0
  160. package/dist/schemas/decision.js +205 -0
  161. package/dist/schemas/download-asset.d.ts +7 -3
  162. package/dist/schemas/engine-addon-map.d.ts +7 -0
  163. package/dist/schemas/engine-addon-map.js +23 -3
  164. package/dist/schemas/errors.d.ts +1 -0
  165. package/dist/schemas/errors.js +5 -0
  166. package/dist/schemas/index.d.ts +2 -0
  167. package/dist/schemas/index.js +2 -0
  168. package/dist/schemas/llamacpp-config.d.ts +42 -2
  169. package/dist/schemas/llamacpp-config.js +27 -0
  170. package/dist/schemas/load-model.d.ts +5247 -1309
  171. package/dist/schemas/load-model.js +88 -9
  172. package/dist/schemas/model-config-schemas.d.ts +233 -8
  173. package/dist/schemas/model-config-schemas.js +4 -1
  174. package/dist/schemas/model-fit-policy.d.ts +17 -0
  175. package/dist/schemas/model-fit-policy.js +17 -0
  176. package/dist/schemas/model-info.d.ts +20 -2
  177. package/dist/schemas/model-info.js +2 -1
  178. package/dist/schemas/model-resource-profile.d.ts +3 -4
  179. package/dist/schemas/model-resource-profile.js +1 -4
  180. package/dist/schemas/model-src-utils.d.ts +10 -2
  181. package/dist/schemas/model-types.d.ts +10 -1
  182. package/dist/schemas/model-types.js +3 -0
  183. package/dist/schemas/ocr.d.ts +74 -0
  184. package/dist/schemas/ocr.js +42 -0
  185. package/dist/schemas/plugin.d.ts +8 -2
  186. package/dist/schemas/plugin.js +6 -0
  187. package/dist/schemas/profiling.d.ts +4 -4
  188. package/dist/schemas/registry.d.ts +26 -0
  189. package/dist/schemas/registry.js +3 -0
  190. package/dist/schemas/reload-config.d.ts +10 -2
  191. package/dist/schemas/rpc-server.js +3 -3
  192. package/dist/schemas/sdcpp-config.d.ts +26 -0
  193. package/dist/schemas/sdcpp-config.js +8 -7
  194. package/dist/schemas/system-resources.d.ts +4 -4
  195. package/dist/schemas/text-to-speech.d.ts +402 -3
  196. package/dist/schemas/text-to-speech.js +129 -11
  197. package/dist/schemas/tools.d.ts +13 -4
  198. package/dist/schemas/tools.js +18 -6
  199. package/dist/schemas/transcription-config.d.ts +15 -0
  200. package/dist/schemas/transcription-config.js +38 -3
  201. package/dist/schemas/transcription.d.ts +46 -4
  202. package/dist/schemas/transcription.js +51 -4
  203. package/dist/schemas/translation-config.d.ts +30 -10
  204. package/dist/schemas/vla.d.ts +9 -2
  205. package/dist/schemas/vla.js +9 -5
  206. package/dist/surface.d.ts +1 -1
  207. package/dist/surface.js +1 -1
  208. package/dist/utils/addon-responses.d.ts +1 -0
  209. package/dist/utils/asr-events.d.ts +1 -0
  210. package/dist/utils/asr-events.js +2 -1
  211. package/dist/utils/cache/types.d.ts +7 -0
  212. package/dist/utils/mcp-adapter.js +12 -8
  213. package/dist/utils/tool-helpers.d.ts +1 -1
  214. package/dist/utils/tool-helpers.js +20 -52
  215. package/dist/utils/tools/parsers/dsml.js +8 -5
  216. package/dist/utils/tools/parsers/qwen35.js +7 -4
  217. package/dist/utils/tools/shared.d.ts +3 -0
  218. package/dist/utils/tools/shared.js +62 -0
  219. package/dist/utils/transcribe-metadata.d.ts +8 -1
  220. package/dist/utils/transcribe-metadata.js +13 -3
  221. package/dist/utils/tts-stats.d.ts +1 -0
  222. package/package.json +41 -32
  223. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
  224. package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
  225. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
  226. package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
  227. package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
  228. package/dist/resources/model-fit/calibration/fit.js +0 -152
  229. package/dist/resources/model-fit/calibration/index.d.ts +0 -37
  230. package/dist/resources/model-fit/calibration/index.js +0 -75
  231. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
  232. package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
  233. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
  234. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
  235. package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
  236. package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
  237. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
  238. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
  239. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
  240. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
  241. package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
  242. package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
  243. package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
  244. package/dist/resources/model-fit/estimators/llm.js +0 -222
  245. package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
  246. package/dist/resources/model-fit/estimators/whisper.js +0 -76
@@ -6,22 +6,19 @@ import { isMobile } from '../../../../runtime/state.js';
6
6
  import { getKVCacheDir } from '../../../../utils/index.js';
7
7
  import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
8
8
  import { getEngineLogger } from '../../../../logging/index.js';
9
- import { z } from 'zod';
10
9
  // Used by cross-model paths that have no `RequestContext` (e.g.
11
10
  // `deleteKvCacheState`). Per-session call sites receive a logger from
12
11
  // the caller — typically `withRequestContext(...)`.
13
12
  const moduleLogger = getEngineLogger();
14
13
  /**
15
- * Coordinates six KV-cache state layers:
14
+ * Coordinates four KV-cache state layers:
16
15
  *
17
- * 1. `cachedPrefixes` — saved message boundaries, and whether a tool
18
- * block was rendered into them.
19
- * 2. `initializedCaches` — caches this process has established.
20
- * 3. On-disk `.bin` files written by the addon.
21
- * 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
22
- * boundary outlives the process; see `readPrefixSidecar`.
23
- * 5. `activeCachePaths` — per-path refs that block in-flight eviction.
24
- * 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
16
+ * 1. `initializedCaches` — caches this process has established.
17
+ * 2. On-disk `.bin` files written by the addon. Each one records the tokens
18
+ * it holds, and the addon reuses the longest prefix the next prompt
19
+ * shares with it, so nothing here tracks what a file covers.
20
+ * 3. `activeCachePaths` — per-path refs that block in-flight eviction.
21
+ * 4. `.auto-cache-<key>` markers — engine-generated cache ownership.
25
22
  *
26
23
  * Every turn must finish through `commitTurn`, `rollback`, or the
27
24
  * non-destructive `releaseTurn` so all inference state stays aligned,
@@ -30,100 +27,6 @@ const moduleLogger = getEngineLogger();
30
27
  */
31
28
  // ----- module-scoped state. The session is the single mutation point
32
29
  // for the in-memory KV-cache bookkeeping. -----
33
- /** What the kv-cache file at a given path is known to hold. */
34
- const cachedPrefixSchema = z.object({
35
- /** Number of chat messages the file on disk is known to cover. */
36
- messages: z.number().int().nonnegative(),
37
- /**
38
- * Whether a static tool block was rendered into that prefix. Tracked
39
- * rather than inferred from `messages`, because a committed turn is not
40
- * proof that its tool block reached the model: the addon drops tools and
41
- * still returns a usable prompt when the chat template rejects them.
42
- */
43
- toolBlock: z.boolean()
44
- });
45
- // The sidecar is written after the addon has already saved the `.bin`, so a
46
- // crash between the two leaves a boundary that describes an older file. The
47
- // `.bin` size is a function of the tokens it holds: it is unchanged when the
48
- // addon re-saves the same state (cancel rewind, key switch) and larger once
49
- // another turn landed in it, which is the case the fingerprint must catch.
50
- const persistedPrefixSchema = cachedPrefixSchema.extend({
51
- binSize: z.number().int().nonnegative()
52
- });
53
- /**
54
- * What the kv-cache file on disk is known to cover, keyed by cache path.
55
- * Written by `commitTurn`, deleted by `rollback`, `delete`,
56
- * `dropStaleSavedCount` and the orphan clear in `beginTurn`. The same
57
- * INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
58
- * present only when the corresponding `.bin` file is considered trustworthy,
59
- * so any turn that leaves the file holding something the boundary does not
60
- * describe has to remove it.
61
- *
62
- * This map outranks the sidecar. Both are written by the same commit, so an
63
- * entry here is never the older copy, and `restorePersistedCache` reads the
64
- * file only to repopulate an empty map after a restart.
65
- */
66
- const cachedPrefixes = new Map();
67
- const PREFIX_SIDECAR_SUFFIX = '.meta.json';
68
- function prefixSidecarPath(cachePath) {
69
- return `${cachePath}${PREFIX_SIDECAR_SUFFIX}`;
70
- }
71
- async function writePrefixSidecar(cachePath, prefix, logger) {
72
- try {
73
- const { size } = await fsPromises.stat(cachePath);
74
- await fsPromises.writeFile(prefixSidecarPath(cachePath), JSON.stringify({ ...prefix, binSize: size }));
75
- }
76
- catch (error) {
77
- // Only the file goes: a sidecar left from the previous commit describes a
78
- // shorter `.bin` than the one now on disk. The in-memory boundary is the
79
- // one this process keeps slicing against and stays.
80
- await removePrefixSidecar(cachePath);
81
- logger.warn(`[kv-cache] Failed to persist saved-message boundary; this process stays warm but a restart will start the cache cold. path=${cachePath} error=${error instanceof Error ? error.message : String(error)}`);
82
- }
83
- }
84
- // Anything unreadable, malformed, or describing a `.bin` of a different size
85
- // is discarded: the cache is then used from a cold boundary, which is the
86
- // pre-sidecar behaviour. Only ever consulted for a path `cachedPrefixes` has
87
- // no entry for, so it can never contradict a live boundary. Never throws: the
88
- // caller holds the cache-path write lock and a `.bin` can vanish under it
89
- // (`deleteKvCacheState` takes no locks).
90
- async function readPrefixSidecar(cachePath) {
91
- let parsed;
92
- let binSize;
93
- try {
94
- const raw = await fsPromises.readFile(prefixSidecarPath(cachePath), 'utf8');
95
- const result = persistedPrefixSchema.safeParse(JSON.parse(raw));
96
- if (!result.success)
97
- return null;
98
- parsed = result.data;
99
- binSize = (await fsPromises.stat(cachePath)).size;
100
- }
101
- catch {
102
- return null;
103
- }
104
- if (binSize !== parsed.binSize) {
105
- // The `.bin` changed after this boundary was recorded, so the file is
106
- // stale whatever wrote it. Only the file: the caller guarantees the map
107
- // holds nothing for this path.
108
- await removePrefixSidecar(cachePath);
109
- return null;
110
- }
111
- return { messages: parsed.messages, toolBlock: parsed.toolBlock };
112
- }
113
- /** Drops the persisted boundary, leaving whatever is in memory alone. */
114
- async function removePrefixSidecar(cachePath) {
115
- try {
116
- await fsPromises.unlink(prefixSidecarPath(cachePath));
117
- }
118
- catch {
119
- // No sidecar for this path.
120
- }
121
- }
122
- /** Drops the boundary everywhere: the cache it describes is gone or untrusted. */
123
- async function forgetPrefix(cachePath) {
124
- cachedPrefixes.delete(cachePath);
125
- await removePrefixSidecar(cachePath);
126
- }
127
30
  /**
128
31
  * In-memory registry of caches established this session, recorded once a save
129
32
  * is verified. The addon defers disk writes, so the absence of a `.bin` file on
@@ -154,7 +57,7 @@ class CacheLockAbortError extends Error {
154
57
  // "session"), which name the SAME file on case-insensitive filesystems (default
155
58
  // macOS/Windows), serialise on one lock and can't interleave writes. Over-locks
156
59
  // case-only variants on case-sensitive filesystems, which is safe. NOTE: the
157
- // other per-path bookkeeping (activeCachePaths, cachedPrefixes, retention)
60
+ // other per-path bookkeeping (activeCachePaths, initializedCaches, retention)
158
61
  // stays case-sensitive, so case-only key variants are otherwise unsupported —
159
62
  // see the KV-cache system doc.
160
63
  function lockKeyFor(cachePath) {
@@ -335,20 +238,24 @@ async function deleteInactiveAutoCaches() {
335
238
  * Construct a session bound to one `(modelId, turn-owning request)`
336
239
  * scope. `options.logger` is the per-instance logger the session emits
337
240
  * through (typically `withRequestContext(getEngineLogger(), ctx)`);
338
- * falls back to the module-scoped logger when omitted.
241
+ * falls back to the module-scoped logger when omitted. `options.discardCache`
242
+ * drops the addon's copy of a conversation whose cache the session rolls
243
+ * back; without it the addon writes that conversation to the deleted path at
244
+ * its next key switch or unload.
339
245
  */
340
246
  export function createKvCacheSession(modelId, options) {
341
247
  const logger = options?.logger ?? moduleLogger;
248
+ const discardCache = options?.discardCache;
342
249
  // Per-session map: each `TurnHandle` carries an opaque entry here. A
343
250
  // WeakMap so handles drop their state once the handler scope releases
344
251
  // the reference; the module-scoped maps above survive.
345
252
  const turnState = new WeakMap();
346
253
  function makeHandle(cachePath, autoCacheKey, releaseWriteLock = () => { }, signal) {
347
- const cached = cachedPrefixes.get(cachePath);
348
254
  const handle = {
349
255
  cachePath,
350
- savedCount: cached?.messages ?? 0,
351
- toolBlockCached: cached?.toolBlock ?? false
256
+ get createdCache() {
257
+ return turnState.get(handle)?.createdByThisTurn ?? false;
258
+ }
352
259
  };
353
260
  turnState.set(handle, {
354
261
  cachePath,
@@ -389,14 +296,10 @@ export function createKvCacheSession(modelId, options) {
389
296
  // In-memory registry check first — the addon defers disk writes, so a
390
297
  // just-saved cache may not yet exist on disk. If the in-memory flag isn't
391
298
  // set, fall back to a filesystem probe so caches surviving across process
392
- // restarts still hit the reuse path. Resolved before the handle is made so
393
- // it snapshots the restored boundary.
299
+ // restarts still hit the reuse path.
394
300
  let exists = initializedCaches.has(cachePath);
395
301
  if (!exists)
396
- exists = await restorePersistedCache(cachePath);
397
- // A boundary without its `.bin` describes a cache that no longer exists.
398
- if (!exists)
399
- await forgetPrefix(cachePath);
302
+ exists = await adoptPersistedCache(cachePath);
400
303
  const handle = makeHandle(cachePath, undefined, releaseWriteLock, input.signal);
401
304
  try {
402
305
  logCacheStatus(input.customKey, exists);
@@ -511,17 +414,13 @@ export function createKvCacheSession(modelId, options) {
511
414
  if (state.committed || state.rolledBack)
512
415
  return;
513
416
  if (result.kind === 'static') {
514
- // Custom-key path: the addon wrote the new cache state inline
515
- // at the same path. Verify the file persisted (the addon
516
- // currently swallows save errors — see TODO in
517
- // `verifySaveAndRecord`) and record the new boundary.
518
- const prefix = { messages: result.messageCount, toolBlock: result.toolBlockCached };
519
- const ok = await verifySaveAndRecord(state.cachePath, prefix);
417
+ // Custom-key path: the addon's save wrote the new cache state at the
418
+ // same path. Verify the file persisted.
419
+ const ok = await verifySaveAndRecord(state.cachePath);
520
420
  if (!ok) {
521
421
  await runRollback(state);
522
422
  return;
523
423
  }
524
- await writePrefixSidecar(state.cachePath, prefix, logger);
525
424
  state.committed = true;
526
425
  releaseCachePath(state.cachePath);
527
426
  state.releaseWriteLock();
@@ -531,13 +430,14 @@ export function createKvCacheSession(modelId, options) {
531
430
  const sourceCacheKey = state.autoCacheKey;
532
431
  const targetCacheKey = path.basename(path.dirname(path.dirname(result.targetCachePath)));
533
432
  // Hold the target's lock across the rename AND commit verification, so a peer
534
- // resolving to the same file can't observe it before its saved count is
535
- // published. Abortable: a cancel while waiting for the lock rejects here.
433
+ // resolving to the same file can't observe it before it is recorded. Abortable: a cancel while waiting for the lock rejects here.
536
434
  const releaseTargetLock = await acquireCachePathWriteLock(result.targetCachePath, state.signal);
537
435
  try {
538
- // A cancel that landed while we waited for the target lock must not commit.
436
+ // A cancel that landed while we waited for the target lock must not
437
+ // commit. The file holds what the addon kept, so it stays unless this
438
+ // turn created it.
539
439
  if (state.signal?.aborted) {
540
- await runRollback(state);
440
+ await runRelease(state);
541
441
  return;
542
442
  }
543
443
  const renamed = await withCacheStateLock(async () => {
@@ -564,7 +464,6 @@ export function createKvCacheSession(modelId, options) {
564
464
  releaseCachePath(sourceCachePath);
565
465
  state.cachePath = result.targetCachePath;
566
466
  state.autoCacheKey = targetCacheKey;
567
- await forgetPrefix(sourceCachePath);
568
467
  await pruneEmptyCacheDirectories(sourceCachePath, snapshotActivePaths());
569
468
  if (sourceCacheKey !== undefined) {
570
469
  await removeAutoCacheMarkerIfMissing(sourceCacheKey);
@@ -579,10 +478,7 @@ export function createKvCacheSession(modelId, options) {
579
478
  await runRollback(state);
580
479
  return;
581
480
  }
582
- const ok = await verifySaveAndRecord(result.targetCachePath, {
583
- messages: result.messageCount,
584
- toolBlock: result.toolBlockCached
585
- });
481
+ const ok = await verifySaveAndRecord(result.targetCachePath);
586
482
  if (!ok) {
587
483
  // Rename succeeded but the file isn't where we expected. Roll back via
588
484
  // the target path instead of the (now-empty) source.
@@ -615,6 +511,9 @@ export function createKvCacheSession(modelId, options) {
615
511
  return;
616
512
  if (state.committed || state.rolledBack)
617
513
  return;
514
+ await runRelease(state);
515
+ }
516
+ async function runRelease(state) {
618
517
  // Nothing committed exists to keep, so a failed first turn is destructive.
619
518
  if (state.createdByThisTurn) {
620
519
  await runRollback(state);
@@ -641,7 +540,14 @@ export function createKvCacheSession(modelId, options) {
641
540
  logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
642
541
  }
643
542
  }
644
- await forgetPrefix(state.cachePath);
543
+ if (discardCache) {
544
+ try {
545
+ await discardCache(state.cachePath);
546
+ }
547
+ catch (discardError) {
548
+ logger.warn(`[kv-cache] Failed to discard the addon's copy of a rolled-back cache; a later unload may write it back. path=${state.cachePath} error=${discardError instanceof Error ? discardError.message : String(discardError)}`);
549
+ }
550
+ }
645
551
  // Release before pruning so an empty parent can go; a sibling still holding
646
552
  // the path keeps it in the active snapshot and protects the directory.
647
553
  releaseCachePath(state.cachePath);
@@ -655,18 +561,11 @@ export function createKvCacheSession(modelId, options) {
655
561
  if (state.autoCacheKey !== undefined)
656
562
  scheduleAutoCacheSweep(logger);
657
563
  }
658
- async function dropStaleSavedCount(turn) {
659
- const state = turnState.get(turn);
660
- if (!state)
661
- return;
662
- await forgetPrefix(state.cachePath);
663
- }
664
564
  return {
665
565
  beginTurn,
666
566
  commitTurn,
667
567
  rollback,
668
- releaseTurn,
669
- dropStaleSavedCount
568
+ releaseTurn
670
569
  };
671
570
  }
672
571
  // ----- module-level administrative API -----
@@ -687,10 +586,8 @@ export function createKvCacheSession(modelId, options) {
687
586
  * Layers cleared, in order:
688
587
  * 1. On-disk: `deleteCache(...)` removes the matching directory
689
588
  * tree (or wipes and recreates the root for `all: true`).
690
- * 2. `cachedPrefixes`: prefix-cleanup by the removed directory
691
- * so any per-cache count under the deleted tree is forgotten.
692
- * 3. `initializedCaches`: prefix-cleanup by the same removed
693
- * directory, since it is keyed by the resolved cache path.
589
+ * 2. `initializedCaches`: prefix-cleanup by the removed directory,
590
+ * since it is keyed by the resolved cache path.
694
591
  *
695
592
  * Concurrency with in-flight turns: this delete does not take the
696
593
  * per-cache-path write locks, so it races any turn holding a `TurnHandle`
@@ -699,9 +596,8 @@ export function createKvCacheSession(modelId, options) {
699
596
  * `Map.delete` / `Set.delete` no-op on absent keys), but the layers are
700
597
  * cleared without a lock, so an interleaving that lands the delete's
701
598
  * in-memory cleanup after a concurrent turn has already renamed/committed
702
- * its file splits state: the file stays on disk while its saved-count and
703
- * init flag are cleared, so the next turn loads the file and reports
704
- * `savedCount=0`. Callers must not delete a key that is in active use.
599
+ * its file splits state: the file stays on disk while its init flag is
600
+ * cleared. Callers must not delete a key that is in active use.
705
601
  * The `auto` target is the exception: it skips keys an in-flight turn holds
706
602
  * and clears each one under the cache-state lock.
707
603
  */
@@ -710,7 +606,6 @@ export async function deleteKvCacheState(target) {
710
606
  return deleteInactiveAutoCaches();
711
607
  if ('all' in target) {
712
608
  const removed = await deleteCacheUtil({ all: true });
713
- cachedPrefixes.clear();
714
609
  initializedCaches.clear();
715
610
  // `removed` is the kv-cache root dir; surfaces it for ops
716
611
  // visibility but isn't part of the contract.
@@ -728,50 +623,40 @@ export async function deleteKvCacheState(target) {
728
623
  // onto the unrelated top-level auto marker.
729
624
  await removeAutoCacheMarker(path.relative(getKVCacheDir(), removedPath));
730
625
  }
731
- // Prefix-cleanup the in-memory counts. The on-disk directory tree
732
- // is `{kvCacheRoot}/{kvCacheKey}[/{modelId}]/`, so every entry in
733
- // `cachedPrefixes` whose key is the removed directory itself
734
- // or sits beneath it must go.
735
- clearCachedMessageCountsByPrefix(removedPath, path.sep);
736
- // initializedCaches is keyed by the resolved cachePath too, so clear it by
737
- // the same removed-directory prefix as cachedPrefixes above.
626
+ // The on-disk directory tree is `{kvCacheRoot}/{kvCacheKey}[/{modelId}]/`,
627
+ // so every entry whose path is the removed directory itself or sits beneath
628
+ // it must go.
738
629
  clearInitializedCachesByPrefix(removedPath, path.sep);
739
630
  }
740
631
  // ----- private helpers -----
741
632
  /**
742
- * Verify the addon actually persisted the cache file before recording
743
- * its message count. The addon currently swallows write errors
744
- * silently, so a missing file means the next turn must resend the full
745
- * history rather than slicing against a stale `savedCount`.
746
- *
747
- * TODO: once the addon surfaces save failures (e.g. throws
748
- * `UnableToSaveSessionFile` when `llama_state_save_file` returns
749
- * false), drop the `access()` probe and wrap the `model.run()` call in
750
- * a real try/catch that forwards the error.
633
+ * Verify the cache file exists before recording it as established. The file
634
+ * is the evidence because `saveCache` rejects both when a write fails and when
635
+ * the addon kept nothing to write.
751
636
  */
752
- async function verifySaveAndRecord(cachePath, prefix) {
637
+ async function verifySaveAndRecord(cachePath) {
753
638
  try {
754
639
  await fsPromises.access(cachePath);
755
- cachedPrefixes.set(cachePath, prefix);
756
640
  initializedCaches.add(cachePath);
757
641
  return true;
758
642
  }
759
643
  catch (err) {
760
- cachedPrefixes.delete(cachePath);
761
644
  logCacheSaveError(cachePath, err);
762
645
  return false;
763
646
  }
764
647
  }
765
648
  /**
766
- * Adopt a `.bin` left by an earlier process run: record it as established and
767
- * pick up the boundary committed alongside it. Returns whether the file exists.
768
- *
769
- * An auto-rename commits its target into `cachedPrefixes` without an init
770
- * flag, so this can run for a path the map already knows. The map wins there
771
- * and the sidecar is left unread, which keeps the two from ever having to be
772
- * reconciled.
649
+ * Adopt a `.bin` left by an earlier process run. Returns whether the file
650
+ * exists. Earlier releases kept a `<bin>.meta.json` boundary sidecar beside
651
+ * it; nothing reads one now, so it goes on first contact.
773
652
  */
774
- async function restorePersistedCache(cachePath) {
653
+ async function adoptPersistedCache(cachePath) {
654
+ try {
655
+ await fsPromises.unlink(`${cachePath}.meta.json`);
656
+ }
657
+ catch {
658
+ // No sidecar for this path.
659
+ }
775
660
  try {
776
661
  await fsPromises.access(cachePath);
777
662
  }
@@ -779,28 +664,8 @@ async function restorePersistedCache(cachePath) {
779
664
  return false;
780
665
  }
781
666
  initializedCaches.add(cachePath);
782
- if (cachedPrefixes.has(cachePath))
783
- return true;
784
- const prefix = await readPrefixSidecar(cachePath);
785
- if (prefix !== null)
786
- cachedPrefixes.set(cachePath, prefix);
787
667
  return true;
788
668
  }
789
- function clearCachedMessageCountsByPrefix(prefix, sep) {
790
- if (!prefix) {
791
- cachedPrefixes.clear();
792
- return;
793
- }
794
- for (const key of cachedPrefixes.keys()) {
795
- if (key === prefix) {
796
- cachedPrefixes.delete(key);
797
- continue;
798
- }
799
- if (!key.startsWith(prefix + sep))
800
- continue;
801
- cachedPrefixes.delete(key);
802
- }
803
- }
804
669
  function clearInitializedCachesByPrefix(prefix, sep) {
805
670
  if (!prefix) {
806
671
  initializedCaches.clear();
@@ -822,21 +687,6 @@ function clearInitializedCachesByPrefix(prefix, sep) {
822
687
  * @internal
823
688
  */
824
689
  export const __kvCacheSessionTestHooks = {
825
- getSavedCount(cachePath) {
826
- return cachedPrefixes.get(cachePath)?.messages;
827
- },
828
- setSavedCountForTest(cachePath, count) {
829
- cachedPrefixes.set(cachePath, { messages: count, toolBlock: false });
830
- },
831
- getToolBlockCachedForTest(cachePath) {
832
- return cachedPrefixes.get(cachePath)?.toolBlock ?? false;
833
- },
834
- getPrefixSidecarPathForTest(cachePath) {
835
- return prefixSidecarPath(cachePath);
836
- },
837
- readPrefixSidecarForTest(cachePath) {
838
- return readPrefixSidecar(cachePath);
839
- },
840
690
  hasInitializedPath(cachePath) {
841
691
  return initializedCaches.has(cachePath);
842
692
  },
@@ -850,7 +700,6 @@ export const __kvCacheSessionTestHooks = {
850
700
  lastAutoCacheSweepMs = value;
851
701
  },
852
702
  resetForTest() {
853
- cachedPrefixes.clear();
854
703
  initializedCaches.clear();
855
704
  activeCachePaths.clear();
856
705
  lastAutoCacheSweepMs = 0;
@@ -1,37 +1,11 @@
1
1
  /**
2
- * Pure decision helper for kv-cache history slicing used by
3
- * `completion-stream.ts` (via the slice computed from
4
- * `TurnHandle.savedCount`).
2
+ * Pure commit decision for kv-cache turns used by `completion-stream.ts`.
5
3
  *
6
4
  * This module intentionally has **no** `bare-*` imports so it can be
7
- * exercised directly from unit tests running under `bun` without
5
+ * exercised directly from unit tests running under Node without
8
6
  * pulling in the Bare runtime (which is not available in that
9
7
  * environment).
10
- *
11
- * Cancel detection flows through the per-request `AbortSignal` from
12
- * `RequestRegistry`; the `cachedMessageCounts` map lives in
13
- * `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
14
- * layers (saved counts, init flags, on-disk files). Only the pure
15
- * slice-decision helper remains here.
16
8
  */
17
- export interface HistoryMessage {
18
- role: string;
19
- content: string;
20
- attachments?: {
21
- path: string;
22
- }[] | undefined;
23
- }
24
- export interface HistorySliceDecision {
25
- /** Messages to send to the model on the next turn. */
26
- messages: HistoryMessage[];
27
- /**
28
- * True when the decision path proves the current `savedCount` is stale
29
- * and the caller should drop the cached entry (via
30
- * `KvCacheSession.dropStaleSavedCount(turn)`) to avoid propagating the
31
- * bad count to the next turn.
32
- */
33
- clearStaleCount: boolean;
34
- }
35
9
  interface CacheCommitContext {
36
10
  aborted: boolean;
37
11
  producedTokens: boolean;
@@ -40,25 +14,4 @@ interface CacheCommitContext {
40
14
  stoppedAtContextBoundary: boolean;
41
15
  }
42
16
  export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
43
- /**
44
- * Pure slice decision for `prepareMessagesForCache`.
45
- *
46
- * Mirrors the shape of the logic in `completion-stream.ts` but without
47
- * calling `transformMessages` (which depends on `bare-fs` for
48
- * attachment probing). Kept here so the decision can be unit-tested in
49
- * isolation.
50
- *
51
- * A non-zero `savedCount` is itself the proof that a cache holds a rendered
52
- * prefix: it is recorded only after a save is verified. It means the prefix
53
- * covers that many messages, so slicing from it drops exactly what the cache
54
- * supplies — the system message among them. Without a usable boundary the
55
- * cache supplies nothing, so the whole history goes to the addon, system
56
- * message included: anything held back would simply be missing from the
57
- * prompt.
58
- *
59
- * The regression guard: a non-zero `savedCount` that slices the history down
60
- * to an empty array is stale, and the caller resends the full history rather
61
- * than handing the model an empty payload.
62
- */
63
- export declare function decideCachedHistorySlice(savedCount: number, history: HistoryMessage[]): HistorySliceDecision;
64
17
  export {};
@@ -1,18 +1,10 @@
1
1
  /**
2
- * Pure decision helper for kv-cache history slicing used by
3
- * `completion-stream.ts` (via the slice computed from
4
- * `TurnHandle.savedCount`).
2
+ * Pure commit decision for kv-cache turns used by `completion-stream.ts`.
5
3
  *
6
4
  * This module intentionally has **no** `bare-*` imports so it can be
7
- * exercised directly from unit tests running under `bun` without
5
+ * exercised directly from unit tests running under Node without
8
6
  * pulling in the Bare runtime (which is not available in that
9
7
  * environment).
10
- *
11
- * Cancel detection flows through the per-request `AbortSignal` from
12
- * `RequestRegistry`; the `cachedMessageCounts` map lives in
13
- * `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
14
- * layers (saved counts, init flags, on-disk files). Only the pure
15
- * slice-decision helper remains here.
16
8
  */
17
9
  export function shouldCommitCachedTurn(context) {
18
10
  const { aborted, producedTokens, generatedTokens, predict, stoppedAtContextBoundary } = context;
@@ -22,39 +14,3 @@ export function shouldCommitCachedTurn(context) {
22
14
  generatedTokens >= predict;
23
15
  return !aborted && producedTokens && !stoppedByBudget && !stoppedAtContextBoundary;
24
16
  }
25
- /**
26
- * Pure slice decision for `prepareMessagesForCache`.
27
- *
28
- * Mirrors the shape of the logic in `completion-stream.ts` but without
29
- * calling `transformMessages` (which depends on `bare-fs` for
30
- * attachment probing). Kept here so the decision can be unit-tested in
31
- * isolation.
32
- *
33
- * A non-zero `savedCount` is itself the proof that a cache holds a rendered
34
- * prefix: it is recorded only after a save is verified. It means the prefix
35
- * covers that many messages, so slicing from it drops exactly what the cache
36
- * supplies — the system message among them. Without a usable boundary the
37
- * cache supplies nothing, so the whole history goes to the addon, system
38
- * message included: anything held back would simply be missing from the
39
- * prompt.
40
- *
41
- * The regression guard: a non-zero `savedCount` that slices the history down
42
- * to an empty array is stale, and the caller resends the full history rather
43
- * than handing the model an empty payload.
44
- */
45
- export function decideCachedHistorySlice(savedCount, history) {
46
- const hasCachedPrefix = savedCount > 0 && history.length > 0;
47
- const sliced = hasCachedPrefix && savedCount <= history.length ? history.slice(savedCount) : null;
48
- // A non-null slice that is empty means the saved count is stale: the
49
- // cached turn boundary is claiming the entire current history is
50
- // already cached, which happens when a previous turn was cancelled
51
- // mid-decode and still recorded `history.length + 1`. Treat it as a
52
- // bad state and resend the full history.
53
- const useSlice = sliced !== null && sliced.length > 0;
54
- return {
55
- messages: useSlice ? sliced : history,
56
- // Only a boundary that was actually consulted can be stale. Absent a
57
- // history to slice, the count is simply unused and left alone.
58
- clearStaleCount: hasCachedPrefix && !useSlice
59
- };
60
- }