@qvac/inference 0.19.0 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +59 -35
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +8 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +53 -13
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +3 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/schemas/assess-model-fit.d.ts +40 -0
- package/dist/schemas/assess-model-fit.js +32 -3
- package/dist/schemas/audio-gen.d.ts +3 -3
- package/dist/schemas/batch-completion-stream.d.ts +3 -0
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +33 -4
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -0
- package/dist/schemas/completion-stream.d.ts +2 -0
- package/dist/schemas/load-model.d.ts +80 -20
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +11 -11
- package/dist/utils/addon-responses.d.ts +1 -0
- package/package.json +2 -2
package/NOTICE
CHANGED
|
@@ -63,14 +63,24 @@ Third-Party Model Licenses
|
|
|
63
63
|
|
|
64
64
|
abot-world-0-5b-lf-dit-q8_0
|
|
65
65
|
https://huggingface.co/acvlab/ABot-World-0-5B-LF
|
|
66
|
+
audio8-codec-decoder-f16
|
|
67
|
+
https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b
|
|
68
|
+
audio8-codec-encoder-f16
|
|
69
|
+
https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b
|
|
70
|
+
audio8-lm-f16
|
|
71
|
+
https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b
|
|
66
72
|
bci-embedder
|
|
67
73
|
https://github.com/tetherto/qvac/releases/download/bci-test-assets-v0.1.0/bci-embedder.bin
|
|
74
|
+
cosyvoice3-campplus-f32
|
|
75
|
+
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
68
76
|
cosyvoice3-flow-f32
|
|
69
77
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
70
78
|
cosyvoice3-hift-f32
|
|
71
79
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
72
80
|
cosyvoice3-llm-q8_0
|
|
73
81
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
82
|
+
cosyvoice3-s3tok-f16
|
|
83
|
+
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
74
84
|
craft_mlt_25k
|
|
75
85
|
https://www.jaided.ai/easyocr/modelhub/
|
|
76
86
|
crnn_mobilenet_v3_small
|
|
@@ -179,6 +189,8 @@ Third-Party Model Licenses
|
|
|
179
189
|
https://huggingface.co/unsloth/Qwen3.6-27B-GGUF
|
|
180
190
|
Qwen3.6-35B-A3B-GGUF
|
|
181
191
|
https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF
|
|
192
|
+
Qwen3.8-27B-GGUF
|
|
193
|
+
https://huggingface.co/unsloth/Qwen3.8-27B-GGUF
|
|
182
194
|
ru-base-ggml-model-f16
|
|
183
195
|
https://huggingface.co/CheeLi03/whisper-base-rus-8
|
|
184
196
|
ru-tiny-ggml-model-f16
|
|
@@ -199,6 +211,10 @@ Third-Party Model Licenses
|
|
|
199
211
|
https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged
|
|
200
212
|
unlimited-ocr-gguf
|
|
201
213
|
https://huggingface.co/vimalnakrani/unlimited-ocr-gguf
|
|
214
|
+
VisionPsy-Nano-460M-Flash-GGUFs
|
|
215
|
+
https://huggingface.co/qvac/VisionPsy-Nano-460M-Flash-GGUFs
|
|
216
|
+
VisionPsy-Nano-460M-GGUFs
|
|
217
|
+
https://huggingface.co/qvac/VisionPsy-Nano-460M-GGUFs
|
|
202
218
|
voice-en
|
|
203
219
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
204
220
|
voice-zh-2
|
|
@@ -348,6 +364,12 @@ Third-Party Model Licenses
|
|
|
348
364
|
https://huggingface.co/ChristianAzinn/gte-large-gguf
|
|
349
365
|
gte-large-gguf
|
|
350
366
|
https://huggingface.co/ChristianAzinn/gte-large-gguf
|
|
367
|
+
indic-conformer-ctc.f16
|
|
368
|
+
https://huggingface.co/ai4bharat/indic-conformer-600m-multilingual
|
|
369
|
+
indic-conformer-ctc.q4_0
|
|
370
|
+
https://huggingface.co/ai4bharat/indic-conformer-600m-multilingual
|
|
371
|
+
indic-conformer-ctc.q8_0
|
|
372
|
+
https://huggingface.co/ai4bharat/indic-conformer-600m-multilingual
|
|
351
373
|
lavasr-denoiser-f16
|
|
352
374
|
https://huggingface.co/spaces/YatharthS
|
|
353
375
|
lavasr-enhancer-f16
|
|
@@ -646,7 +668,7 @@ JavaScript Dependencies
|
|
|
646
668
|
@qvac/error@0.1.1
|
|
647
669
|
@qvac/logging@0.1.1
|
|
648
670
|
https://github.com/tetherto/qvac
|
|
649
|
-
@qvac/rag@0.
|
|
671
|
+
@qvac/rag@0.8.1
|
|
650
672
|
https://github.com/tetherto/qvac
|
|
651
673
|
@qvac/registry-client@0.6.1
|
|
652
674
|
https://github.com/tetherto/qvac
|
|
@@ -666,47 +688,49 @@ JavaScript Dependencies
|
|
|
666
688
|
https://github.com/holepunchto/bare-ansi-escapes
|
|
667
689
|
bare-assert@1.2.0
|
|
668
690
|
https://github.com/holepunchto/bare-assert
|
|
669
|
-
bare-buffer@3.7.
|
|
691
|
+
bare-buffer@3.7.1
|
|
670
692
|
https://github.com/holepunchto/bare-buffer
|
|
671
693
|
bare-cpu-info@0.1.1
|
|
672
694
|
https://github.com/holepunchto/bare-cpu-info
|
|
673
695
|
bare-crypto@1.15.3
|
|
674
696
|
https://github.com/holepunchto/bare-crypto
|
|
675
|
-
bare-dns@2.
|
|
697
|
+
bare-dns@2.2.0
|
|
676
698
|
https://github.com/holepunchto/bare-dns
|
|
677
699
|
bare-env@3.0.1
|
|
678
700
|
https://github.com/holepunchto/bare-env
|
|
679
|
-
bare-events@2.9.
|
|
701
|
+
bare-events@2.9.2
|
|
680
702
|
https://github.com/holepunchto/bare-events
|
|
703
|
+
bare-fetch@3.2.1
|
|
704
|
+
https://github.com/holepunchto/bare-fetch
|
|
681
705
|
bare-form-data@1.2.2
|
|
682
706
|
https://github.com/holepunchto/bare-form-data
|
|
683
|
-
bare-fs@4.8.
|
|
707
|
+
bare-fs@4.8.1
|
|
684
708
|
https://github.com/holepunchto/bare-fs
|
|
685
709
|
bare-gpu-info@0.1.1
|
|
686
710
|
https://github.com/holepunchto/bare-gpu-info
|
|
687
|
-
bare-hrtime@2.1.
|
|
711
|
+
bare-hrtime@2.1.2
|
|
688
712
|
https://github.com/holepunchto/bare-hrtime
|
|
689
|
-
bare-http-parser@
|
|
713
|
+
bare-http-parser@2.1.4
|
|
690
714
|
https://github.com/holepunchto/bare-http-parser
|
|
691
|
-
bare-http1@4.
|
|
715
|
+
bare-http1@4.6.1
|
|
692
716
|
https://github.com/holepunchto/bare-http1
|
|
693
|
-
bare-https@3.
|
|
717
|
+
bare-https@3.1.0
|
|
694
718
|
https://github.com/holepunchto/bare-https
|
|
695
|
-
bare-inspect@3.1.
|
|
719
|
+
bare-inspect@3.1.10
|
|
696
720
|
https://github.com/holepunchto/bare-inspect
|
|
697
721
|
bare-mime@1.0.0
|
|
698
722
|
https://github.com/holepunchto/bare-mime
|
|
699
|
-
bare-module-resolve@1.12.
|
|
723
|
+
bare-module-resolve@1.12.5
|
|
700
724
|
https://github.com/holepunchto/bare-module-resolve
|
|
701
725
|
bare-net@2.3.3
|
|
702
726
|
https://github.com/holepunchto/bare-net
|
|
703
727
|
bare-os@3.9.3
|
|
704
728
|
https://github.com/holepunchto/bare-os
|
|
705
|
-
bare-path@3.1.
|
|
729
|
+
bare-path@3.1.2
|
|
706
730
|
https://github.com/holepunchto/bare-path
|
|
707
731
|
bare-performance@2.1.1
|
|
708
732
|
https://github.com/holepunchto/bare-performance
|
|
709
|
-
bare-pipe@4.
|
|
733
|
+
bare-pipe@4.3.1
|
|
710
734
|
https://github.com/holepunchto/bare-pipe
|
|
711
735
|
bare-posix@1.0.1
|
|
712
736
|
https://github.com/holepunchto/bare-posix
|
|
@@ -720,23 +744,23 @@ JavaScript Dependencies
|
|
|
720
744
|
https://github.com/holepunchto/bare-signals
|
|
721
745
|
bare-stdio@1.0.3
|
|
722
746
|
https://github.com/holepunchto/bare-stdio
|
|
723
|
-
bare-stream@2.13.
|
|
747
|
+
bare-stream@2.13.4
|
|
724
748
|
https://github.com/holepunchto/bare-stream
|
|
725
|
-
bare-tcp@2.
|
|
749
|
+
bare-tcp@2.6.1
|
|
726
750
|
https://github.com/holepunchto/bare-tcp
|
|
727
|
-
bare-tls@3.1.
|
|
751
|
+
bare-tls@3.1.10
|
|
728
752
|
https://github.com/holepunchto/bare-tls
|
|
729
|
-
bare-tty@5.1
|
|
753
|
+
bare-tty@5.2.1
|
|
730
754
|
https://github.com/holepunchto/bare-tty
|
|
731
|
-
bare-type@1.1.
|
|
755
|
+
bare-type@1.1.1
|
|
732
756
|
https://github.com/holepunchto/bare-type
|
|
733
|
-
bare-url@2.5.
|
|
757
|
+
bare-url@2.5.4
|
|
734
758
|
https://github.com/holepunchto/bare-url
|
|
735
759
|
bare-zlib@1.4.1
|
|
736
760
|
https://github.com/holepunchto/bare-zlib
|
|
737
761
|
blind-relay@1.6.1
|
|
738
762
|
https://github.com/holepunchto/blind-relay
|
|
739
|
-
compact-encoding@3.
|
|
763
|
+
compact-encoding@3.5.0
|
|
740
764
|
https://github.com/holepunchto/compact-encoding
|
|
741
765
|
compact-encoding-bitfield@1.1.0
|
|
742
766
|
https://github.com/compact-encoding/compact-encoding-bitfield
|
|
@@ -748,7 +772,7 @@ JavaScript Dependencies
|
|
|
748
772
|
https://github.com/holepunchto/events-universal
|
|
749
773
|
fd-lock@2.2.0
|
|
750
774
|
https://github.com/holepunchto/fd-lock
|
|
751
|
-
fs-native-extensions@1.5.
|
|
775
|
+
fs-native-extensions@1.5.1
|
|
752
776
|
https://github.com/holepunchto/fs-native-extensions
|
|
753
777
|
hyperblobs@2.12.1
|
|
754
778
|
https://github.com/holepunchto/hyperblobs
|
|
@@ -760,17 +784,17 @@ JavaScript Dependencies
|
|
|
760
784
|
https://github.com/holepunchto/hypercore-stats
|
|
761
785
|
hypercore-storage@3.2.1
|
|
762
786
|
https://github.com/holepunchto/hypercore-storage
|
|
763
|
-
hyperdb@6.
|
|
787
|
+
hyperdb@6.9.0
|
|
764
788
|
https://github.com/holepunchto/hyperdb
|
|
765
789
|
hyperdht-address@1.1.1
|
|
766
790
|
https://github.com/holepunchto/hyperdht-address
|
|
767
|
-
hyperdht-stats@1.
|
|
791
|
+
hyperdht-stats@1.11.0
|
|
768
792
|
https://github.com/holepunchto/hyperdht-stats
|
|
769
793
|
hyperdispatch@1.6.0
|
|
770
794
|
https://github.com/holepunchto/hyperdispatch
|
|
771
795
|
hyperdrive@13.3.3
|
|
772
796
|
https://github.com/holepunchto/hyperdrive
|
|
773
|
-
hyperschema@1.
|
|
797
|
+
hyperschema@1.24.0
|
|
774
798
|
https://github.com/holepunchto/hyperschema
|
|
775
799
|
hyperswarm-stats@1.3.1
|
|
776
800
|
https://github.com/holepunchto/hyperswarm-stats
|
|
@@ -780,7 +804,7 @@ JavaScript Dependencies
|
|
|
780
804
|
https://github.com/holepunchto/mirror-drive
|
|
781
805
|
noise-handshake@4.2.0
|
|
782
806
|
https://github.com/holepunchto/noise-handshake
|
|
783
|
-
paparam@1.
|
|
807
|
+
paparam@1.13.0
|
|
784
808
|
https://github.com/holepunchto/paparam
|
|
785
809
|
passive-core-watcher@1.0.1
|
|
786
810
|
https://github.com/holepunchto/passive-core-watcher
|
|
@@ -794,7 +818,7 @@ JavaScript Dependencies
|
|
|
794
818
|
https://github.com/holepunchto/rache
|
|
795
819
|
refcounter@1.0.0
|
|
796
820
|
https://github.com/holepunchto/refcounter
|
|
797
|
-
require-addon@1.
|
|
821
|
+
require-addon@1.3.0
|
|
798
822
|
https://github.com/holepunchto/require-addon
|
|
799
823
|
resource-on-exit@1.0.0
|
|
800
824
|
https://github.com/holepunchto/bare-teardown
|
|
@@ -808,7 +832,7 @@ JavaScript Dependencies
|
|
|
808
832
|
https://github.com/holepunchto/sub-encoder
|
|
809
833
|
text-decoder@1.2.7
|
|
810
834
|
https://github.com/holepunchto/text-decoder
|
|
811
|
-
udx-native@1.21.
|
|
835
|
+
udx-native@1.21.1
|
|
812
836
|
https://github.com/holepunchto/udx-native
|
|
813
837
|
unslab@1.3.0
|
|
814
838
|
https://github.com/holepunchto/unslab
|
|
@@ -840,7 +864,7 @@ JavaScript Dependencies
|
|
|
840
864
|
https://github.com/mafintosh/bogon
|
|
841
865
|
codecs@3.1.0
|
|
842
866
|
https://github.com/mafintosh/codecs
|
|
843
|
-
corestore@7.12.
|
|
867
|
+
corestore@7.12.4
|
|
844
868
|
https://github.com/holepunchto/corestore
|
|
845
869
|
debounceify@1.1.0
|
|
846
870
|
https://github.com/mafintosh/debounceify
|
|
@@ -858,13 +882,13 @@ JavaScript Dependencies
|
|
|
858
882
|
https://github.com/mafintosh/generate-string
|
|
859
883
|
hyperbee@2.27.3
|
|
860
884
|
https://github.com/holepunchto/hyperbee
|
|
861
|
-
hypercore@11.35.
|
|
885
|
+
hypercore@11.35.5
|
|
862
886
|
https://github.com/holepunchto/hypercore
|
|
863
887
|
hypercore-crypto@3.7.0
|
|
864
888
|
https://github.com/mafintosh/hypercore-crypto
|
|
865
|
-
hyperdht@6.
|
|
889
|
+
hyperdht@6.34.0
|
|
866
890
|
https://github.com/holepunchto/hyperdht
|
|
867
|
-
hyperswarm@4.17.
|
|
891
|
+
hyperswarm@4.17.1
|
|
868
892
|
https://github.com/holepunchto/hyperswarm
|
|
869
893
|
is-options@1.0.2
|
|
870
894
|
https://github.com/mafintosh/is-options
|
|
@@ -880,7 +904,7 @@ JavaScript Dependencies
|
|
|
880
904
|
https://github.com/mafintosh/nat-sampler
|
|
881
905
|
protocol-buffers-encodings@1.2.0
|
|
882
906
|
https://github.com/mafintosh/protocol-buffers-encodings
|
|
883
|
-
protomux@3.
|
|
907
|
+
protomux@3.12.0
|
|
884
908
|
https://github.com/holepunchto/protomux
|
|
885
909
|
queue-tick@1.0.1
|
|
886
910
|
https://github.com/mafintosh/queue-tick
|
|
@@ -910,9 +934,9 @@ JavaScript Dependencies
|
|
|
910
934
|
https://github.com/holepunchto/sodium-universal
|
|
911
935
|
speedometer@1.1.0
|
|
912
936
|
https://github.com/mafintosh/speedometer
|
|
913
|
-
streamx@2.28.
|
|
937
|
+
streamx@2.28.1
|
|
914
938
|
https://github.com/mafintosh/streamx
|
|
915
|
-
tar-stream@3.2.
|
|
939
|
+
tar-stream@3.2.1
|
|
916
940
|
https://github.com/mafintosh/tar-stream
|
|
917
941
|
teex@1.0.1
|
|
918
942
|
https://github.com/mafintosh/teex
|
|
@@ -932,6 +956,6 @@ JavaScript Dependencies
|
|
|
932
956
|
https://github.com/mafintosh/xache
|
|
933
957
|
z32@1.1.0
|
|
934
958
|
https://github.com/mafintosh/z32
|
|
935
|
-
zod@4.
|
|
959
|
+
zod@4.6.1
|
|
936
960
|
https://github.com/colinhacks/zod
|
|
937
961
|
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* Deletes KV cache files.
|
|
3
3
|
*
|
|
4
4
|
* @param params - The delete cache parameters
|
|
5
|
-
* @param params.all - If true, deletes all cache files
|
|
5
|
+
* @param params.all - If true, deletes all cache files, including caller-owned named caches
|
|
6
|
+
* @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
|
|
6
7
|
* @param params.kvCacheKey - The cache key to delete
|
|
7
8
|
* @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
|
|
8
9
|
* @returns Promise resolving to success status
|
|
@@ -12,6 +13,9 @@
|
|
|
12
13
|
* // Delete all caches
|
|
13
14
|
* await deleteCache({ all: true });
|
|
14
15
|
*
|
|
16
|
+
* // Reclaim disk from automatic caches, leaving named caches alone
|
|
17
|
+
* await deleteCache({ auto: true });
|
|
18
|
+
*
|
|
15
19
|
* // Delete entire cache key (all models)
|
|
16
20
|
* await deleteCache({ kvCacheKey: "my-session" });
|
|
17
21
|
*
|
|
@@ -21,6 +25,8 @@
|
|
|
21
25
|
*/
|
|
22
26
|
export declare function deleteCache(params: {
|
|
23
27
|
all: true;
|
|
28
|
+
} | {
|
|
29
|
+
auto: true;
|
|
24
30
|
} | {
|
|
25
31
|
kvCacheKey: string;
|
|
26
32
|
modelId?: string;
|
package/dist/api/delete-cache.js
CHANGED
|
@@ -4,7 +4,8 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors
|
|
|
4
4
|
* Deletes KV cache files.
|
|
5
5
|
*
|
|
6
6
|
* @param params - The delete cache parameters
|
|
7
|
-
* @param params.all - If true, deletes all cache files
|
|
7
|
+
* @param params.all - If true, deletes all cache files, including caller-owned named caches
|
|
8
|
+
* @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
|
|
8
9
|
* @param params.kvCacheKey - The cache key to delete
|
|
9
10
|
* @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
|
|
10
11
|
* @returns Promise resolving to success status
|
|
@@ -14,6 +15,9 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors
|
|
|
14
15
|
* // Delete all caches
|
|
15
16
|
* await deleteCache({ all: true });
|
|
16
17
|
*
|
|
18
|
+
* // Reclaim disk from automatic caches, leaving named caches alone
|
|
19
|
+
* await deleteCache({ auto: true });
|
|
20
|
+
*
|
|
17
21
|
* // Delete entire cache key (all models)
|
|
18
22
|
* await deleteCache({ kvCacheKey: "my-session" });
|
|
19
23
|
*
|
|
@@ -29,6 +33,12 @@ export async function deleteCache(params) {
|
|
|
29
33
|
all: true
|
|
30
34
|
};
|
|
31
35
|
}
|
|
36
|
+
else if ('auto' in params && params.auto) {
|
|
37
|
+
req = {
|
|
38
|
+
type: 'deleteCache',
|
|
39
|
+
auto: true
|
|
40
|
+
};
|
|
41
|
+
}
|
|
32
42
|
else if ('kvCacheKey' in params) {
|
|
33
43
|
req = {
|
|
34
44
|
type: 'deleteCache',
|
|
@@ -15,6 +15,9 @@ export async function handleDeleteCache(request) {
|
|
|
15
15
|
if ('all' in request && request.all) {
|
|
16
16
|
await deleteKvCacheState({ all: true });
|
|
17
17
|
}
|
|
18
|
+
else if ('auto' in request && request.auto) {
|
|
19
|
+
await deleteKvCacheState({ auto: true });
|
|
20
|
+
}
|
|
18
21
|
else if ('kvCacheKey' in request) {
|
|
19
22
|
await deleteKvCacheState({
|
|
20
23
|
kvCacheKey: request.kvCacheKey,
|
|
@@ -3,6 +3,7 @@ import type { LlmStats } from '../../../../utils/addon-responses.js';
|
|
|
3
3
|
export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
|
|
4
4
|
timeToFirstToken?: number | undefined;
|
|
5
5
|
tokensPerSecond?: number | undefined;
|
|
6
|
+
promptTokensPerSecond?: number | undefined;
|
|
6
7
|
cacheTokens?: number | undefined;
|
|
7
8
|
promptTokens?: number | undefined;
|
|
8
9
|
generatedTokens?: number | undefined;
|
|
@@ -6,6 +6,7 @@ export function normalizeCompletionStats(stats) {
|
|
|
6
6
|
return undefined;
|
|
7
7
|
const timeToFirstToken = finiteNumber(stats.TTFT);
|
|
8
8
|
const tokensPerSecond = finiteNumber(stats.TPS);
|
|
9
|
+
const promptTokensPerSecond = finiteNumber(stats.ppTPS);
|
|
9
10
|
const cacheTokens = finiteNumber(stats.CacheTokens);
|
|
10
11
|
const promptTokens = finiteNumber(stats.promptTokens);
|
|
11
12
|
const generatedTokens = finiteNumber(stats.generatedTokens);
|
|
@@ -13,6 +14,7 @@ export function normalizeCompletionStats(stats) {
|
|
|
13
14
|
const normalized = {
|
|
14
15
|
...(timeToFirstToken !== undefined && { timeToFirstToken }),
|
|
15
16
|
...(tokensPerSecond !== undefined && { tokensPerSecond }),
|
|
17
|
+
...(promptTokensPerSecond !== undefined && { promptTokensPerSecond }),
|
|
16
18
|
...(cacheTokens !== undefined && { cacheTokens }),
|
|
17
19
|
...(promptTokens !== undefined && { promptTokens }),
|
|
18
20
|
...(generatedTokens !== undefined && { generatedTokens }),
|
|
@@ -21,6 +23,7 @@ export function normalizeCompletionStats(stats) {
|
|
|
21
23
|
};
|
|
22
24
|
if (timeToFirstToken === undefined &&
|
|
23
25
|
tokensPerSecond === undefined &&
|
|
26
|
+
promptTokensPerSecond === undefined &&
|
|
24
27
|
cacheTokens === undefined &&
|
|
25
28
|
promptTokens === undefined &&
|
|
26
29
|
generatedTokens === undefined &&
|
|
@@ -132,9 +132,10 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
132
132
|
}): KvCacheSession;
|
|
133
133
|
/**
|
|
134
134
|
* Atomically delete every layer of KV-cache state for a
|
|
135
|
-
* `(kvCacheKey, modelId)` pair,
|
|
136
|
-
*
|
|
137
|
-
* turn-scoped
|
|
135
|
+
* `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
|
|
136
|
+
* caches no turn is holding. Single entry point — the only mutation
|
|
137
|
+
* point for cross-model state outside of turn-scoped
|
|
138
|
+
* `commitTurn`/`rollback`.
|
|
138
139
|
*
|
|
139
140
|
* Why this isn't a method on `KvCacheSession`: deletes are
|
|
140
141
|
* cross-model (`all: true` has no model; the keyed form has
|
|
@@ -161,12 +162,16 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
161
162
|
* its file splits state: the file stays on disk while its saved-count and
|
|
162
163
|
* init flag are cleared, so the next turn sees the file, skips priming, and
|
|
163
164
|
* reports `savedCount=0`. Callers must not delete a key that is in active use.
|
|
165
|
+
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
166
|
+
* and clears each one under the cache-state lock.
|
|
164
167
|
*/
|
|
165
168
|
export declare function deleteKvCacheState(target: {
|
|
166
169
|
kvCacheKey: string;
|
|
167
170
|
modelId?: string;
|
|
168
171
|
} | {
|
|
169
172
|
all: true;
|
|
173
|
+
} | {
|
|
174
|
+
auto: true;
|
|
170
175
|
}): Promise<void>;
|
|
171
176
|
/**
|
|
172
177
|
* Test-only access to the module-scoped state. Production code reaches
|
|
@@ -128,6 +128,25 @@ function isCacheKeyActive(cacheKey) {
|
|
|
128
128
|
function getAutoCacheMaxBytes() {
|
|
129
129
|
return isMobile() ? MOBILE_AUTO_CACHE_MAX_BYTES : DESKTOP_AUTO_CACHE_MAX_BYTES;
|
|
130
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* Deletes planned auto-cache keys and reports how many went.
|
|
133
|
+
*
|
|
134
|
+
* Planning and eviction are separate steps, so a turn can take a key in
|
|
135
|
+
* between: each one is re-checked against the active paths under the
|
|
136
|
+
* cache-state lock rather than trusted from the plan.
|
|
137
|
+
*/
|
|
138
|
+
async function evictPlannedAutoCaches(cacheKeys) {
|
|
139
|
+
let evicted = 0;
|
|
140
|
+
await withCacheStateLock(async () => {
|
|
141
|
+
for (const cacheKey of cacheKeys) {
|
|
142
|
+
if (isCacheKeyActive(cacheKey))
|
|
143
|
+
continue;
|
|
144
|
+
await deleteKvCacheState({ kvCacheKey: cacheKey });
|
|
145
|
+
evicted++;
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
return evicted;
|
|
149
|
+
}
|
|
131
150
|
async function withCacheStateLock(operation) {
|
|
132
151
|
const previous = cacheStateLockTail;
|
|
133
152
|
let releaseLock = () => { };
|
|
@@ -156,20 +175,12 @@ async function maybeSweepAutoCaches(logger, overrides) {
|
|
|
156
175
|
const sweep = async () => {
|
|
157
176
|
try {
|
|
158
177
|
const cacheKeys = await planAutoCacheEvictions({
|
|
159
|
-
activeCachePaths:
|
|
178
|
+
activeCachePaths: snapshotActivePaths(),
|
|
160
179
|
maxBytes: overrides?.maxBytes ?? getAutoCacheMaxBytes(),
|
|
161
180
|
maxIdleMs: overrides?.maxIdleMs ?? AUTO_CACHE_MAX_IDLE_MS,
|
|
162
181
|
nowMs
|
|
163
182
|
});
|
|
164
|
-
|
|
165
|
-
await withCacheStateLock(async () => {
|
|
166
|
-
for (const cacheKey of cacheKeys) {
|
|
167
|
-
if (isCacheKeyActive(cacheKey))
|
|
168
|
-
continue;
|
|
169
|
-
await deleteKvCacheState({ kvCacheKey: cacheKey });
|
|
170
|
-
evictionCount++;
|
|
171
|
-
}
|
|
172
|
-
});
|
|
183
|
+
const evictionCount = await evictPlannedAutoCaches(cacheKeys);
|
|
173
184
|
if (evictionCount > 0) {
|
|
174
185
|
logger.debug(`[kv-cache] Evicted ${evictionCount} inactive auto-cache entries`);
|
|
175
186
|
}
|
|
@@ -192,6 +203,30 @@ function scheduleAutoCacheSweep(logger) {
|
|
|
192
203
|
logger.warn(`[kv-cache] Failed to schedule auto-cache retention sweep: ${error instanceof Error ? error.message : String(error)}`);
|
|
193
204
|
});
|
|
194
205
|
}
|
|
206
|
+
/**
|
|
207
|
+
* Drops every auto cache no turn is holding, whatever its age or size.
|
|
208
|
+
*
|
|
209
|
+
* The on-demand counterpart to `maybeSweepAutoCaches`, which applies the
|
|
210
|
+
* standing quota and TTL instead. Reusing the planner keeps the "is this an
|
|
211
|
+
* auto cache" decision in one place: only `.auto-cache-<key>` markers are
|
|
212
|
+
* considered, so caller-owned named caches stay out of scope.
|
|
213
|
+
*/
|
|
214
|
+
async function deleteInactiveAutoCaches() {
|
|
215
|
+
const cacheKeys = await planAutoCacheEvictions({
|
|
216
|
+
activeCachePaths: snapshotActivePaths(),
|
|
217
|
+
// The quota pass stops at `retainedBytes <= maxBytes`, and a total reaches
|
|
218
|
+
// 0 while zero-byte caches remain (an empty `.bin` from a crashed prime).
|
|
219
|
+
// A negative quota is unreachable, so every inactive entry is selected.
|
|
220
|
+
maxBytes: -1,
|
|
221
|
+
// No age threshold: the quota pass above already selects everything.
|
|
222
|
+
maxIdleMs: 0,
|
|
223
|
+
nowMs: Date.now()
|
|
224
|
+
});
|
|
225
|
+
const evicted = await evictPlannedAutoCaches(cacheKeys);
|
|
226
|
+
if (evicted > 0) {
|
|
227
|
+
moduleLogger.debug(`[kv-cache] Reclaimed ${evicted} inactive auto-cache entries`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
195
230
|
// ----- factory -----
|
|
196
231
|
/**
|
|
197
232
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
@@ -548,9 +583,10 @@ export function createKvCacheSession(modelId, options) {
|
|
|
548
583
|
// ----- module-level administrative API -----
|
|
549
584
|
/**
|
|
550
585
|
* Atomically delete every layer of KV-cache state for a
|
|
551
|
-
* `(kvCacheKey, modelId)` pair,
|
|
552
|
-
*
|
|
553
|
-
* turn-scoped
|
|
586
|
+
* `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
|
|
587
|
+
* caches no turn is holding. Single entry point — the only mutation
|
|
588
|
+
* point for cross-model state outside of turn-scoped
|
|
589
|
+
* `commitTurn`/`rollback`.
|
|
554
590
|
*
|
|
555
591
|
* Why this isn't a method on `KvCacheSession`: deletes are
|
|
556
592
|
* cross-model (`all: true` has no model; the keyed form has
|
|
@@ -577,8 +613,12 @@ export function createKvCacheSession(modelId, options) {
|
|
|
577
613
|
* its file splits state: the file stays on disk while its saved-count and
|
|
578
614
|
* init flag are cleared, so the next turn sees the file, skips priming, and
|
|
579
615
|
* reports `savedCount=0`. Callers must not delete a key that is in active use.
|
|
616
|
+
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
617
|
+
* and clears each one under the cache-state lock.
|
|
580
618
|
*/
|
|
581
619
|
export async function deleteKvCacheState(target) {
|
|
620
|
+
if ('auto' in target)
|
|
621
|
+
return deleteInactiveAutoCaches();
|
|
582
622
|
if ('all' in target) {
|
|
583
623
|
const removed = await deleteCacheUtil({ all: true });
|
|
584
624
|
cachedPrefixes.clear();
|
|
@@ -291,6 +291,7 @@ export declare const llmPlugin: {
|
|
|
291
291
|
stats: import("zod").ZodObject<{
|
|
292
292
|
timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
293
293
|
tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
294
|
+
promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
294
295
|
cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
295
296
|
promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
296
297
|
generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
@@ -328,6 +329,7 @@ export declare const llmPlugin: {
|
|
|
328
329
|
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
329
330
|
timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
330
331
|
tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
332
|
+
promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
331
333
|
cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
332
334
|
promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
333
335
|
generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
@@ -451,6 +453,7 @@ export declare const llmPlugin: {
|
|
|
451
453
|
stats: import("zod").ZodObject<{
|
|
452
454
|
timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
453
455
|
tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
456
|
+
promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
454
457
|
cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
455
458
|
promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
456
459
|
generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|