@qvac/inference 0.19.0 → 0.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/NOTICE +59 -35
  2. package/dist/api/delete-cache.d.ts +7 -1
  3. package/dist/api/delete-cache.js +11 -1
  4. package/dist/handlers/delete-cache.js +3 -0
  5. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  6. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  7. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +8 -3
  8. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +53 -13
  9. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +3 -0
  10. package/dist/resources/model-fit/assess.js +169 -68
  11. package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
  12. package/dist/resources/model-fit/estimators/llm.js +3 -1
  13. package/dist/resources/model-fit/floor.d.ts +31 -0
  14. package/dist/resources/model-fit/floor.js +65 -0
  15. package/dist/resources/native.js +5 -4
  16. package/dist/resources/types.d.ts +1 -3
  17. package/dist/schemas/assess-model-fit.d.ts +40 -0
  18. package/dist/schemas/assess-model-fit.js +32 -3
  19. package/dist/schemas/audio-gen.d.ts +3 -3
  20. package/dist/schemas/batch-completion-stream.d.ts +3 -0
  21. package/dist/schemas/bci.d.ts +1 -1
  22. package/dist/schemas/common.d.ts +33 -4
  23. package/dist/schemas/completion-event.d.ts +3 -0
  24. package/dist/schemas/completion-event.js +1 -0
  25. package/dist/schemas/completion-stream.d.ts +2 -0
  26. package/dist/schemas/load-model.d.ts +80 -20
  27. package/dist/schemas/model-ops.d.ts +3 -0
  28. package/dist/schemas/model-ops.js +6 -0
  29. package/dist/schemas/plugin.d.ts +11 -11
  30. package/dist/utils/addon-responses.d.ts +1 -0
  31. package/package.json +2 -2
package/NOTICE CHANGED
@@ -63,14 +63,24 @@ Third-Party Model Licenses
63
63
 
64
64
  abot-world-0-5b-lf-dit-q8_0
65
65
  https://huggingface.co/acvlab/ABot-World-0-5B-LF
66
+ audio8-codec-decoder-f16
67
+ https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b
68
+ audio8-codec-encoder-f16
69
+ https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b
70
+ audio8-lm-f16
71
+ https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b
66
72
  bci-embedder
67
73
  https://github.com/tetherto/qvac/releases/download/bci-test-assets-v0.1.0/bci-embedder.bin
74
+ cosyvoice3-campplus-f32
75
+ https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
68
76
  cosyvoice3-flow-f32
69
77
  https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
70
78
  cosyvoice3-hift-f32
71
79
  https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
72
80
  cosyvoice3-llm-q8_0
73
81
  https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
82
+ cosyvoice3-s3tok-f16
83
+ https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
74
84
  craft_mlt_25k
75
85
  https://www.jaided.ai/easyocr/modelhub/
76
86
  crnn_mobilenet_v3_small
@@ -179,6 +189,8 @@ Third-Party Model Licenses
179
189
  https://huggingface.co/unsloth/Qwen3.6-27B-GGUF
180
190
  Qwen3.6-35B-A3B-GGUF
181
191
  https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF
192
+ Qwen3.8-27B-GGUF
193
+ https://huggingface.co/unsloth/Qwen3.8-27B-GGUF
182
194
  ru-base-ggml-model-f16
183
195
  https://huggingface.co/CheeLi03/whisper-base-rus-8
184
196
  ru-tiny-ggml-model-f16
@@ -199,6 +211,10 @@ Third-Party Model Licenses
199
211
  https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged
200
212
  unlimited-ocr-gguf
201
213
  https://huggingface.co/vimalnakrani/unlimited-ocr-gguf
214
+ VisionPsy-Nano-460M-Flash-GGUFs
215
+ https://huggingface.co/qvac/VisionPsy-Nano-460M-Flash-GGUFs
216
+ VisionPsy-Nano-460M-GGUFs
217
+ https://huggingface.co/qvac/VisionPsy-Nano-460M-GGUFs
202
218
  voice-en
203
219
  https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
204
220
  voice-zh-2
@@ -348,6 +364,12 @@ Third-Party Model Licenses
348
364
  https://huggingface.co/ChristianAzinn/gte-large-gguf
349
365
  gte-large-gguf
350
366
  https://huggingface.co/ChristianAzinn/gte-large-gguf
367
+ indic-conformer-ctc.f16
368
+ https://huggingface.co/ai4bharat/indic-conformer-600m-multilingual
369
+ indic-conformer-ctc.q4_0
370
+ https://huggingface.co/ai4bharat/indic-conformer-600m-multilingual
371
+ indic-conformer-ctc.q8_0
372
+ https://huggingface.co/ai4bharat/indic-conformer-600m-multilingual
351
373
  lavasr-denoiser-f16
352
374
  https://huggingface.co/spaces/YatharthS
353
375
  lavasr-enhancer-f16
@@ -646,7 +668,7 @@ JavaScript Dependencies
646
668
  @qvac/error@0.1.1
647
669
  @qvac/logging@0.1.1
648
670
  https://github.com/tetherto/qvac
649
- @qvac/rag@0.6.4
671
+ @qvac/rag@0.8.1
650
672
  https://github.com/tetherto/qvac
651
673
  @qvac/registry-client@0.6.1
652
674
  https://github.com/tetherto/qvac
@@ -666,47 +688,49 @@ JavaScript Dependencies
666
688
  https://github.com/holepunchto/bare-ansi-escapes
667
689
  bare-assert@1.2.0
668
690
  https://github.com/holepunchto/bare-assert
669
- bare-buffer@3.7.0
691
+ bare-buffer@3.7.1
670
692
  https://github.com/holepunchto/bare-buffer
671
693
  bare-cpu-info@0.1.1
672
694
  https://github.com/holepunchto/bare-cpu-info
673
695
  bare-crypto@1.15.3
674
696
  https://github.com/holepunchto/bare-crypto
675
- bare-dns@2.1.4
697
+ bare-dns@2.2.0
676
698
  https://github.com/holepunchto/bare-dns
677
699
  bare-env@3.0.1
678
700
  https://github.com/holepunchto/bare-env
679
- bare-events@2.9.1
701
+ bare-events@2.9.2
680
702
  https://github.com/holepunchto/bare-events
703
+ bare-fetch@3.2.1
704
+ https://github.com/holepunchto/bare-fetch
681
705
  bare-form-data@1.2.2
682
706
  https://github.com/holepunchto/bare-form-data
683
- bare-fs@4.8.0
707
+ bare-fs@4.8.1
684
708
  https://github.com/holepunchto/bare-fs
685
709
  bare-gpu-info@0.1.1
686
710
  https://github.com/holepunchto/bare-gpu-info
687
- bare-hrtime@2.1.1
711
+ bare-hrtime@2.1.2
688
712
  https://github.com/holepunchto/bare-hrtime
689
- bare-http-parser@1.1.5
713
+ bare-http-parser@2.1.4
690
714
  https://github.com/holepunchto/bare-http-parser
691
- bare-http1@4.5.8
715
+ bare-http1@4.6.1
692
716
  https://github.com/holepunchto/bare-http1
693
- bare-https@3.0.0
717
+ bare-https@3.1.0
694
718
  https://github.com/holepunchto/bare-https
695
- bare-inspect@3.1.5
719
+ bare-inspect@3.1.10
696
720
  https://github.com/holepunchto/bare-inspect
697
721
  bare-mime@1.0.0
698
722
  https://github.com/holepunchto/bare-mime
699
- bare-module-resolve@1.12.4
723
+ bare-module-resolve@1.12.5
700
724
  https://github.com/holepunchto/bare-module-resolve
701
725
  bare-net@2.3.3
702
726
  https://github.com/holepunchto/bare-net
703
727
  bare-os@3.9.3
704
728
  https://github.com/holepunchto/bare-os
705
- bare-path@3.1.1
729
+ bare-path@3.1.2
706
730
  https://github.com/holepunchto/bare-path
707
731
  bare-performance@2.1.1
708
732
  https://github.com/holepunchto/bare-performance
709
- bare-pipe@4.2.3
733
+ bare-pipe@4.3.1
710
734
  https://github.com/holepunchto/bare-pipe
711
735
  bare-posix@1.0.1
712
736
  https://github.com/holepunchto/bare-posix
@@ -720,23 +744,23 @@ JavaScript Dependencies
720
744
  https://github.com/holepunchto/bare-signals
721
745
  bare-stdio@1.0.3
722
746
  https://github.com/holepunchto/bare-stdio
723
- bare-stream@2.13.3
747
+ bare-stream@2.13.4
724
748
  https://github.com/holepunchto/bare-stream
725
- bare-tcp@2.5.4
749
+ bare-tcp@2.6.1
726
750
  https://github.com/holepunchto/bare-tcp
727
- bare-tls@3.1.8
751
+ bare-tls@3.1.10
728
752
  https://github.com/holepunchto/bare-tls
729
- bare-tty@5.1.2
753
+ bare-tty@5.2.1
730
754
  https://github.com/holepunchto/bare-tty
731
- bare-type@1.1.0
755
+ bare-type@1.1.1
732
756
  https://github.com/holepunchto/bare-type
733
- bare-url@2.5.2
757
+ bare-url@2.5.4
734
758
  https://github.com/holepunchto/bare-url
735
759
  bare-zlib@1.4.1
736
760
  https://github.com/holepunchto/bare-zlib
737
761
  blind-relay@1.6.1
738
762
  https://github.com/holepunchto/blind-relay
739
- compact-encoding@3.3.0
763
+ compact-encoding@3.5.0
740
764
  https://github.com/holepunchto/compact-encoding
741
765
  compact-encoding-bitfield@1.1.0
742
766
  https://github.com/compact-encoding/compact-encoding-bitfield
@@ -748,7 +772,7 @@ JavaScript Dependencies
748
772
  https://github.com/holepunchto/events-universal
749
773
  fd-lock@2.2.0
750
774
  https://github.com/holepunchto/fd-lock
751
- fs-native-extensions@1.5.0
775
+ fs-native-extensions@1.5.1
752
776
  https://github.com/holepunchto/fs-native-extensions
753
777
  hyperblobs@2.12.1
754
778
  https://github.com/holepunchto/hyperblobs
@@ -760,17 +784,17 @@ JavaScript Dependencies
760
784
  https://github.com/holepunchto/hypercore-stats
761
785
  hypercore-storage@3.2.1
762
786
  https://github.com/holepunchto/hypercore-storage
763
- hyperdb@6.8.0
787
+ hyperdb@6.9.0
764
788
  https://github.com/holepunchto/hyperdb
765
789
  hyperdht-address@1.1.1
766
790
  https://github.com/holepunchto/hyperdht-address
767
- hyperdht-stats@1.10.0
791
+ hyperdht-stats@1.11.0
768
792
  https://github.com/holepunchto/hyperdht-stats
769
793
  hyperdispatch@1.6.0
770
794
  https://github.com/holepunchto/hyperdispatch
771
795
  hyperdrive@13.3.3
772
796
  https://github.com/holepunchto/hyperdrive
773
- hyperschema@1.22.0
797
+ hyperschema@1.24.0
774
798
  https://github.com/holepunchto/hyperschema
775
799
  hyperswarm-stats@1.3.1
776
800
  https://github.com/holepunchto/hyperswarm-stats
@@ -780,7 +804,7 @@ JavaScript Dependencies
780
804
  https://github.com/holepunchto/mirror-drive
781
805
  noise-handshake@4.2.0
782
806
  https://github.com/holepunchto/noise-handshake
783
- paparam@1.12.0
807
+ paparam@1.13.0
784
808
  https://github.com/holepunchto/paparam
785
809
  passive-core-watcher@1.0.1
786
810
  https://github.com/holepunchto/passive-core-watcher
@@ -794,7 +818,7 @@ JavaScript Dependencies
794
818
  https://github.com/holepunchto/rache
795
819
  refcounter@1.0.0
796
820
  https://github.com/holepunchto/refcounter
797
- require-addon@1.2.0
821
+ require-addon@1.3.0
798
822
  https://github.com/holepunchto/require-addon
799
823
  resource-on-exit@1.0.0
800
824
  https://github.com/holepunchto/bare-teardown
@@ -808,7 +832,7 @@ JavaScript Dependencies
808
832
  https://github.com/holepunchto/sub-encoder
809
833
  text-decoder@1.2.7
810
834
  https://github.com/holepunchto/text-decoder
811
- udx-native@1.21.0
835
+ udx-native@1.21.1
812
836
  https://github.com/holepunchto/udx-native
813
837
  unslab@1.3.0
814
838
  https://github.com/holepunchto/unslab
@@ -840,7 +864,7 @@ JavaScript Dependencies
840
864
  https://github.com/mafintosh/bogon
841
865
  codecs@3.1.0
842
866
  https://github.com/mafintosh/codecs
843
- corestore@7.12.0
867
+ corestore@7.12.4
844
868
  https://github.com/holepunchto/corestore
845
869
  debounceify@1.1.0
846
870
  https://github.com/mafintosh/debounceify
@@ -858,13 +882,13 @@ JavaScript Dependencies
858
882
  https://github.com/mafintosh/generate-string
859
883
  hyperbee@2.27.3
860
884
  https://github.com/holepunchto/hyperbee
861
- hypercore@11.35.1
885
+ hypercore@11.35.5
862
886
  https://github.com/holepunchto/hypercore
863
887
  hypercore-crypto@3.7.0
864
888
  https://github.com/mafintosh/hypercore-crypto
865
- hyperdht@6.33.1
889
+ hyperdht@6.34.0
866
890
  https://github.com/holepunchto/hyperdht
867
- hyperswarm@4.17.0
891
+ hyperswarm@4.17.1
868
892
  https://github.com/holepunchto/hyperswarm
869
893
  is-options@1.0.2
870
894
  https://github.com/mafintosh/is-options
@@ -880,7 +904,7 @@ JavaScript Dependencies
880
904
  https://github.com/mafintosh/nat-sampler
881
905
  protocol-buffers-encodings@1.2.0
882
906
  https://github.com/mafintosh/protocol-buffers-encodings
883
- protomux@3.11.0
907
+ protomux@3.12.0
884
908
  https://github.com/holepunchto/protomux
885
909
  queue-tick@1.0.1
886
910
  https://github.com/mafintosh/queue-tick
@@ -910,9 +934,9 @@ JavaScript Dependencies
910
934
  https://github.com/holepunchto/sodium-universal
911
935
  speedometer@1.1.0
912
936
  https://github.com/mafintosh/speedometer
913
- streamx@2.28.0
937
+ streamx@2.28.1
914
938
  https://github.com/mafintosh/streamx
915
- tar-stream@3.2.0
939
+ tar-stream@3.2.1
916
940
  https://github.com/mafintosh/tar-stream
917
941
  teex@1.0.1
918
942
  https://github.com/mafintosh/teex
@@ -932,6 +956,6 @@ JavaScript Dependencies
932
956
  https://github.com/mafintosh/xache
933
957
  z32@1.1.0
934
958
  https://github.com/mafintosh/z32
935
- zod@4.4.3
959
+ zod@4.6.1
936
960
  https://github.com/colinhacks/zod
937
961
 
@@ -2,7 +2,8 @@
2
2
  * Deletes KV cache files.
3
3
  *
4
4
  * @param params - The delete cache parameters
5
- * @param params.all - If true, deletes all cache files
5
+ * @param params.all - If true, deletes all cache files, including caller-owned named caches
6
+ * @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
6
7
  * @param params.kvCacheKey - The cache key to delete
7
8
  * @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
8
9
  * @returns Promise resolving to success status
@@ -12,6 +13,9 @@
12
13
  * // Delete all caches
13
14
  * await deleteCache({ all: true });
14
15
  *
16
+ * // Reclaim disk from automatic caches, leaving named caches alone
17
+ * await deleteCache({ auto: true });
18
+ *
15
19
  * // Delete entire cache key (all models)
16
20
  * await deleteCache({ kvCacheKey: "my-session" });
17
21
  *
@@ -21,6 +25,8 @@
21
25
  */
22
26
  export declare function deleteCache(params: {
23
27
  all: true;
28
+ } | {
29
+ auto: true;
24
30
  } | {
25
31
  kvCacheKey: string;
26
32
  modelId?: string;
@@ -4,7 +4,8 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors
4
4
  * Deletes KV cache files.
5
5
  *
6
6
  * @param params - The delete cache parameters
7
- * @param params.all - If true, deletes all cache files
7
+ * @param params.all - If true, deletes all cache files, including caller-owned named caches
8
+ * @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
8
9
  * @param params.kvCacheKey - The cache key to delete
9
10
  * @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
10
11
  * @returns Promise resolving to success status
@@ -14,6 +15,9 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors
14
15
  * // Delete all caches
15
16
  * await deleteCache({ all: true });
16
17
  *
18
+ * // Reclaim disk from automatic caches, leaving named caches alone
19
+ * await deleteCache({ auto: true });
20
+ *
17
21
  * // Delete entire cache key (all models)
18
22
  * await deleteCache({ kvCacheKey: "my-session" });
19
23
  *
@@ -29,6 +33,12 @@ export async function deleteCache(params) {
29
33
  all: true
30
34
  };
31
35
  }
36
+ else if ('auto' in params && params.auto) {
37
+ req = {
38
+ type: 'deleteCache',
39
+ auto: true
40
+ };
41
+ }
32
42
  else if ('kvCacheKey' in params) {
33
43
  req = {
34
44
  type: 'deleteCache',
@@ -15,6 +15,9 @@ export async function handleDeleteCache(request) {
15
15
  if ('all' in request && request.all) {
16
16
  await deleteKvCacheState({ all: true });
17
17
  }
18
+ else if ('auto' in request && request.auto) {
19
+ await deleteKvCacheState({ auto: true });
20
+ }
18
21
  else if ('kvCacheKey' in request) {
19
22
  await deleteKvCacheState({
20
23
  kvCacheKey: request.kvCacheKey,
@@ -3,6 +3,7 @@ import type { LlmStats } from '../../../../utils/addon-responses.js';
3
3
  export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
4
4
  timeToFirstToken?: number | undefined;
5
5
  tokensPerSecond?: number | undefined;
6
+ promptTokensPerSecond?: number | undefined;
6
7
  cacheTokens?: number | undefined;
7
8
  promptTokens?: number | undefined;
8
9
  generatedTokens?: number | undefined;
@@ -6,6 +6,7 @@ export function normalizeCompletionStats(stats) {
6
6
  return undefined;
7
7
  const timeToFirstToken = finiteNumber(stats.TTFT);
8
8
  const tokensPerSecond = finiteNumber(stats.TPS);
9
+ const promptTokensPerSecond = finiteNumber(stats.ppTPS);
9
10
  const cacheTokens = finiteNumber(stats.CacheTokens);
10
11
  const promptTokens = finiteNumber(stats.promptTokens);
11
12
  const generatedTokens = finiteNumber(stats.generatedTokens);
@@ -13,6 +14,7 @@ export function normalizeCompletionStats(stats) {
13
14
  const normalized = {
14
15
  ...(timeToFirstToken !== undefined && { timeToFirstToken }),
15
16
  ...(tokensPerSecond !== undefined && { tokensPerSecond }),
17
+ ...(promptTokensPerSecond !== undefined && { promptTokensPerSecond }),
16
18
  ...(cacheTokens !== undefined && { cacheTokens }),
17
19
  ...(promptTokens !== undefined && { promptTokens }),
18
20
  ...(generatedTokens !== undefined && { generatedTokens }),
@@ -21,6 +23,7 @@ export function normalizeCompletionStats(stats) {
21
23
  };
22
24
  if (timeToFirstToken === undefined &&
23
25
  tokensPerSecond === undefined &&
26
+ promptTokensPerSecond === undefined &&
24
27
  cacheTokens === undefined &&
25
28
  promptTokens === undefined &&
26
29
  generatedTokens === undefined &&
@@ -132,9 +132,10 @@ export declare function createKvCacheSession(modelId: string, options?: {
132
132
  }): KvCacheSession;
133
133
  /**
134
134
  * Atomically delete every layer of KV-cache state for a
135
- * `(kvCacheKey, modelId)` pair, or wipe everything. Single entry point
136
- * — the only mutation point for cross-model state outside of
137
- * turn-scoped `commitTurn`/`rollback`.
135
+ * `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
136
+ * caches no turn is holding. Single entry point — the only mutation
137
+ * point for cross-model state outside of turn-scoped
138
+ * `commitTurn`/`rollback`.
138
139
  *
139
140
  * Why this isn't a method on `KvCacheSession`: deletes are
140
141
  * cross-model (`all: true` has no model; the keyed form has
@@ -161,12 +162,16 @@ export declare function createKvCacheSession(modelId: string, options?: {
161
162
  * its file splits state: the file stays on disk while its saved-count and
162
163
  * init flag are cleared, so the next turn sees the file, skips priming, and
163
164
  * reports `savedCount=0`. Callers must not delete a key that is in active use.
165
+ * The `auto` target is the exception: it skips keys an in-flight turn holds
166
+ * and clears each one under the cache-state lock.
164
167
  */
165
168
  export declare function deleteKvCacheState(target: {
166
169
  kvCacheKey: string;
167
170
  modelId?: string;
168
171
  } | {
169
172
  all: true;
173
+ } | {
174
+ auto: true;
170
175
  }): Promise<void>;
171
176
  /**
172
177
  * Test-only access to the module-scoped state. Production code reaches
@@ -128,6 +128,25 @@ function isCacheKeyActive(cacheKey) {
128
128
  function getAutoCacheMaxBytes() {
129
129
  return isMobile() ? MOBILE_AUTO_CACHE_MAX_BYTES : DESKTOP_AUTO_CACHE_MAX_BYTES;
130
130
  }
131
+ /**
132
+ * Deletes planned auto-cache keys and reports how many went.
133
+ *
134
+ * Planning and eviction are separate steps, so a turn can take a key in
135
+ * between: each one is re-checked against the active paths under the
136
+ * cache-state lock rather than trusted from the plan.
137
+ */
138
+ async function evictPlannedAutoCaches(cacheKeys) {
139
+ let evicted = 0;
140
+ await withCacheStateLock(async () => {
141
+ for (const cacheKey of cacheKeys) {
142
+ if (isCacheKeyActive(cacheKey))
143
+ continue;
144
+ await deleteKvCacheState({ kvCacheKey: cacheKey });
145
+ evicted++;
146
+ }
147
+ });
148
+ return evicted;
149
+ }
131
150
  async function withCacheStateLock(operation) {
132
151
  const previous = cacheStateLockTail;
133
152
  let releaseLock = () => { };
@@ -156,20 +175,12 @@ async function maybeSweepAutoCaches(logger, overrides) {
156
175
  const sweep = async () => {
157
176
  try {
158
177
  const cacheKeys = await planAutoCacheEvictions({
159
- activeCachePaths: Array.from(activeCachePaths.keys()),
178
+ activeCachePaths: snapshotActivePaths(),
160
179
  maxBytes: overrides?.maxBytes ?? getAutoCacheMaxBytes(),
161
180
  maxIdleMs: overrides?.maxIdleMs ?? AUTO_CACHE_MAX_IDLE_MS,
162
181
  nowMs
163
182
  });
164
- let evictionCount = 0;
165
- await withCacheStateLock(async () => {
166
- for (const cacheKey of cacheKeys) {
167
- if (isCacheKeyActive(cacheKey))
168
- continue;
169
- await deleteKvCacheState({ kvCacheKey: cacheKey });
170
- evictionCount++;
171
- }
172
- });
183
+ const evictionCount = await evictPlannedAutoCaches(cacheKeys);
173
184
  if (evictionCount > 0) {
174
185
  logger.debug(`[kv-cache] Evicted ${evictionCount} inactive auto-cache entries`);
175
186
  }
@@ -192,6 +203,30 @@ function scheduleAutoCacheSweep(logger) {
192
203
  logger.warn(`[kv-cache] Failed to schedule auto-cache retention sweep: ${error instanceof Error ? error.message : String(error)}`);
193
204
  });
194
205
  }
206
+ /**
207
+ * Drops every auto cache no turn is holding, whatever its age or size.
208
+ *
209
+ * The on-demand counterpart to `maybeSweepAutoCaches`, which applies the
210
+ * standing quota and TTL instead. Reusing the planner keeps the "is this an
211
+ * auto cache" decision in one place: only `.auto-cache-<key>` markers are
212
+ * considered, so caller-owned named caches stay out of scope.
213
+ */
214
+ async function deleteInactiveAutoCaches() {
215
+ const cacheKeys = await planAutoCacheEvictions({
216
+ activeCachePaths: snapshotActivePaths(),
217
+ // The quota pass stops at `retainedBytes <= maxBytes`, and a total reaches
218
+ // 0 while zero-byte caches remain (an empty `.bin` from a crashed prime).
219
+ // A negative quota is unreachable, so every inactive entry is selected.
220
+ maxBytes: -1,
221
+ // No age threshold: the quota pass above already selects everything.
222
+ maxIdleMs: 0,
223
+ nowMs: Date.now()
224
+ });
225
+ const evicted = await evictPlannedAutoCaches(cacheKeys);
226
+ if (evicted > 0) {
227
+ moduleLogger.debug(`[kv-cache] Reclaimed ${evicted} inactive auto-cache entries`);
228
+ }
229
+ }
195
230
  // ----- factory -----
196
231
  /**
197
232
  * Construct a session bound to one `(modelId, turn-owning request)`
@@ -548,9 +583,10 @@ export function createKvCacheSession(modelId, options) {
548
583
  // ----- module-level administrative API -----
549
584
  /**
550
585
  * Atomically delete every layer of KV-cache state for a
551
- * `(kvCacheKey, modelId)` pair, or wipe everything. Single entry point
552
- * — the only mutation point for cross-model state outside of
553
- * turn-scoped `commitTurn`/`rollback`.
586
+ * `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
587
+ * caches no turn is holding. Single entry point — the only mutation
588
+ * point for cross-model state outside of turn-scoped
589
+ * `commitTurn`/`rollback`.
554
590
  *
555
591
  * Why this isn't a method on `KvCacheSession`: deletes are
556
592
  * cross-model (`all: true` has no model; the keyed form has
@@ -577,8 +613,12 @@ export function createKvCacheSession(modelId, options) {
577
613
  * its file splits state: the file stays on disk while its saved-count and
578
614
  * init flag are cleared, so the next turn sees the file, skips priming, and
579
615
  * reports `savedCount=0`. Callers must not delete a key that is in active use.
616
+ * The `auto` target is the exception: it skips keys an in-flight turn holds
617
+ * and clears each one under the cache-state lock.
580
618
  */
581
619
  export async function deleteKvCacheState(target) {
620
+ if ('auto' in target)
621
+ return deleteInactiveAutoCaches();
582
622
  if ('all' in target) {
583
623
  const removed = await deleteCacheUtil({ all: true });
584
624
  cachedPrefixes.clear();
@@ -291,6 +291,7 @@ export declare const llmPlugin: {
291
291
  stats: import("zod").ZodObject<{
292
292
  timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
293
293
  tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
294
+ promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
294
295
  cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
295
296
  promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
296
297
  generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
@@ -328,6 +329,7 @@ export declare const llmPlugin: {
328
329
  stats: import("zod").ZodOptional<import("zod").ZodObject<{
329
330
  timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
330
331
  tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
332
+ promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
331
333
  cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
332
334
  promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
333
335
  generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
@@ -451,6 +453,7 @@ export declare const llmPlugin: {
451
453
  stats: import("zod").ZodObject<{
452
454
  timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
453
455
  tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
456
+ promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
454
457
  cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
455
458
  promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
456
459
  generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;