@qvac/inference 0.0.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +179 -0
- package/NOTICE +928 -0
- package/README.md +138 -0
- package/dist/api/audio-gen.d.ts +18 -0
- package/dist/api/audio-gen.js +148 -0
- package/dist/api/batch-completion.d.ts +14 -0
- package/dist/api/batch-completion.js +332 -0
- package/dist/api/bci-transcribe.d.ts +56 -0
- package/dist/api/bci-transcribe.js +172 -0
- package/dist/api/cancel.d.ts +34 -0
- package/dist/api/cancel.js +82 -0
- package/dist/api/classify.d.ts +22 -0
- package/dist/api/classify.js +46 -0
- package/dist/api/completion-stream.d.ts +106 -0
- package/dist/api/completion-stream.js +351 -0
- package/dist/api/delete-cache.d.ts +29 -0
- package/dist/api/delete-cache.js +49 -0
- package/dist/api/diffusion.d.ts +92 -0
- package/dist/api/diffusion.js +176 -0
- package/dist/api/download-asset.d.ts +49 -0
- package/dist/api/download-asset.js +87 -0
- package/dist/api/embed.d.ts +41 -0
- package/dist/api/embed.js +29 -0
- package/dist/api/finetune.d.ts +89 -0
- package/dist/api/finetune.js +138 -0
- package/dist/api/get-loaded-model-info.d.ts +24 -0
- package/dist/api/get-loaded-model-info.js +34 -0
- package/dist/api/get-model-info.d.ts +43 -0
- package/dist/api/get-model-info.js +21 -0
- package/dist/api/get-system-resources.d.ts +2 -0
- package/dist/api/get-system-resources.js +16 -0
- package/dist/api/heartbeat.d.ts +29 -0
- package/dist/api/heartbeat.js +37 -0
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +35 -0
- package/dist/api/invoke-plugin.d.ts +30 -0
- package/dist/api/invoke-plugin.js +54 -0
- package/dist/api/load-model.d.ts +169 -0
- package/dist/api/load-model.js +129 -0
- package/dist/api/logging-stream-registry.d.ts +4 -0
- package/dist/api/logging-stream-registry.js +61 -0
- package/dist/api/logging-stream.d.ts +23 -0
- package/dist/api/logging-stream.js +36 -0
- package/dist/api/ocr.d.ts +30 -0
- package/dist/api/ocr.js +88 -0
- package/dist/api/parse-input.d.ts +2 -0
- package/dist/api/parse-input.js +14 -0
- package/dist/api/provide.d.ts +23 -0
- package/dist/api/provide.js +32 -0
- package/dist/api/rag.d.ts +251 -0
- package/dist/api/rag.js +485 -0
- package/dist/api/registry.d.ts +39 -0
- package/dist/api/registry.js +63 -0
- package/dist/api/resume.d.ts +29 -0
- package/dist/api/resume.js +36 -0
- package/dist/api/state.d.ts +22 -0
- package/dist/api/state.js +29 -0
- package/dist/api/stop-provider.d.ts +20 -0
- package/dist/api/stop-provider.js +30 -0
- package/dist/api/subscribe-logs.d.ts +27 -0
- package/dist/api/subscribe-logs.js +43 -0
- package/dist/api/suspend.d.ts +30 -0
- package/dist/api/suspend.js +37 -0
- package/dist/api/text-to-speech.d.ts +64 -0
- package/dist/api/text-to-speech.js +434 -0
- package/dist/api/transcribe.d.ts +75 -0
- package/dist/api/transcribe.js +256 -0
- package/dist/api/translate.d.ts +47 -0
- package/dist/api/translate.js +115 -0
- package/dist/api/unload-model.d.ts +17 -0
- package/dist/api/unload-model.js +42 -0
- package/dist/api/upscale.d.ts +44 -0
- package/dist/api/upscale.js +99 -0
- package/dist/api/video.d.ts +86 -0
- package/dist/api/video.js +176 -0
- package/dist/api/vla-helpers.d.ts +43 -0
- package/dist/api/vla-helpers.js +124 -0
- package/dist/api/vla.d.ts +68 -0
- package/dist/api/vla.js +113 -0
- package/dist/config/config-utils.d.ts +4 -0
- package/dist/config/config-utils.js +18 -0
- package/dist/config/resolve-config.d.ts +8 -0
- package/dist/config/resolve-config.js +74 -0
- package/dist/constants/audio.d.ts +1 -0
- package/dist/constants/audio.js +1 -0
- package/dist/constants/index.d.ts +2 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/registry.d.ts +1 -0
- package/dist/constants/registry.js +1 -0
- package/dist/dispatch.d.ts +18 -0
- package/dist/dispatch.js +271 -0
- package/dist/engine.d.ts +5 -0
- package/dist/engine.js +27 -0
- package/dist/errors/index.d.ts +416 -0
- package/dist/errors/index.js +681 -0
- package/dist/errors/options.d.ts +2 -0
- package/dist/errors/options.js +13 -0
- package/dist/handlers/cancelHandler.d.ts +29 -0
- package/dist/handlers/cancelHandler.js +67 -0
- package/dist/handlers/completion-orchestrate.d.ts +41 -0
- package/dist/handlers/completion-orchestrate.js +190 -0
- package/dist/handlers/delete-cache.d.ts +11 -0
- package/dist/handlers/delete-cache.js +37 -0
- package/dist/handlers/download-asset.d.ts +2 -0
- package/dist/handlers/download-asset.js +146 -0
- package/dist/handlers/finetune.d.ts +2 -0
- package/dist/handlers/finetune.js +18 -0
- package/dist/handlers/get-loaded-model-info.d.ts +2 -0
- package/dist/handlers/get-loaded-model-info.js +45 -0
- package/dist/handlers/get-model-info.d.ts +2 -0
- package/dist/handlers/get-model-info.js +173 -0
- package/dist/handlers/get-system-resources.d.ts +2 -0
- package/dist/handlers/get-system-resources.js +53 -0
- package/dist/handlers/heartbeat.d.ts +2 -0
- package/dist/handlers/heartbeat.js +3 -0
- package/dist/handlers/load-model/download-manager.d.ts +95 -0
- package/dist/handlers/load-model/download-manager.js +306 -0
- package/dist/handlers/load-model/download-stats.d.ts +7 -0
- package/dist/handlers/load-model/download-stats.js +126 -0
- package/dist/handlers/load-model/handler.d.ts +2 -0
- package/dist/handlers/load-model/handler.js +287 -0
- package/dist/handlers/load-model/http.d.ts +3 -0
- package/dist/handlers/load-model/http.js +857 -0
- package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
- package/dist/handlers/load-model/hyperdrive.js +560 -0
- package/dist/handlers/load-model/index.d.ts +2 -0
- package/dist/handlers/load-model/index.js +2 -0
- package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/handlers/load-model/registry-client-options.js +18 -0
- package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/handlers/load-model/registry-companion-set.js +206 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
- package/dist/handlers/load-model/registry-download-utils.js +112 -0
- package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
- package/dist/handlers/load-model/registry-metadata.js +23 -0
- package/dist/handlers/load-model/registry.d.ts +4 -0
- package/dist/handlers/load-model/registry.js +298 -0
- package/dist/handlers/load-model/resolve-session.d.ts +33 -0
- package/dist/handlers/load-model/resolve-session.js +62 -0
- package/dist/handlers/load-model/resolve.d.ts +5 -0
- package/dist/handlers/load-model/resolve.js +158 -0
- package/dist/handlers/load-model/types.d.ts +59 -0
- package/dist/handlers/load-model/types.js +28 -0
- package/dist/handlers/logging-stream.d.ts +2 -0
- package/dist/handlers/logging-stream.js +39 -0
- package/dist/handlers/plugin-dispatch.d.ts +3 -0
- package/dist/handlers/plugin-dispatch.js +63 -0
- package/dist/handlers/plugin-invoke.d.ts +3 -0
- package/dist/handlers/plugin-invoke.js +81 -0
- package/dist/handlers/registry.d.ts +4 -0
- package/dist/handlers/registry.js +156 -0
- package/dist/handlers/resume.d.ts +2 -0
- package/dist/handlers/resume.js +11 -0
- package/dist/handlers/state.d.ts +2 -0
- package/dist/handlers/state.js +7 -0
- package/dist/handlers/suspend.d.ts +2 -0
- package/dist/handlers/suspend.js +11 -0
- package/dist/handlers/types.d.ts +13 -0
- package/dist/handlers/types.js +1 -0
- package/dist/handlers/unload-model.d.ts +2 -0
- package/dist/handlers/unload-model.js +29 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +12 -0
- package/dist/logging/addon.d.ts +5 -0
- package/dist/logging/addon.js +75 -0
- package/dist/logging/index.d.ts +5 -0
- package/dist/logging/index.js +4 -0
- package/dist/logging/logger.d.ts +41 -0
- package/dist/logging/logger.js +209 -0
- package/dist/logging/namespaces.d.ts +6 -0
- package/dist/logging/namespaces.js +7 -0
- package/dist/logging/types.d.ts +37 -0
- package/dist/logging/types.js +1 -0
- package/dist/logging/utils.d.ts +5 -0
- package/dist/logging/utils.js +113 -0
- package/dist/models/registry/index.d.ts +18 -0
- package/dist/models/registry/index.js +21 -0
- package/dist/models/registry/models.d.ts +22680 -0
- package/dist/models/registry/models.js +23908 -0
- package/dist/p2p/cancel-delegated.d.ts +3 -0
- package/dist/p2p/cancel-delegated.js +64 -0
- package/dist/p2p/completion-stream-delegated.d.ts +4 -0
- package/dist/p2p/completion-stream-delegated.js +51 -0
- package/dist/p2p/delegate-client.d.ts +9 -0
- package/dist/p2p/delegate-client.js +219 -0
- package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
- package/dist/p2p/delegate-connect-diagnostics.js +38 -0
- package/dist/p2p/delegate-transport.d.ts +18 -0
- package/dist/p2p/delegate-transport.js +237 -0
- package/dist/p2p/error-handlers.d.ts +4 -0
- package/dist/p2p/error-handlers.js +31 -0
- package/dist/p2p/handle-request.d.ts +2 -0
- package/dist/p2p/handle-request.js +171 -0
- package/dist/p2p/handler-utils.d.ts +19 -0
- package/dist/p2p/handler-utils.js +180 -0
- package/dist/p2p/heartbeat-delegated.d.ts +3 -0
- package/dist/p2p/heartbeat-delegated.js +32 -0
- package/dist/p2p/load-model-delegated.d.ts +6 -0
- package/dist/p2p/load-model-delegated.js +132 -0
- package/dist/p2p/progress-throttle.d.ts +7 -0
- package/dist/p2p/progress-throttle.js +47 -0
- package/dist/p2p/provideHandler/connection.d.ts +2 -0
- package/dist/p2p/provideHandler/connection.js +38 -0
- package/dist/p2p/provideHandler/handler.d.ts +2 -0
- package/dist/p2p/provideHandler/handler.js +66 -0
- package/dist/p2p/provideHandler/proxy.d.ts +2 -0
- package/dist/p2p/provideHandler/proxy.js +24 -0
- package/dist/p2p/provideHandler.d.ts +1 -0
- package/dist/p2p/provideHandler.js +2 -0
- package/dist/p2p/rpc-utils.d.ts +4 -0
- package/dist/p2p/rpc-utils.js +8 -0
- package/dist/p2p/stopProvideHandler.d.ts +2 -0
- package/dist/p2p/stopProvideHandler.js +29 -0
- package/dist/p2p/swarm.d.ts +9 -0
- package/dist/p2p/swarm.js +103 -0
- package/dist/p2p/transport-selector.d.ts +2 -0
- package/dist/p2p/transport-selector.js +18 -0
- package/dist/p2p/unload-model-delegated.d.ts +2 -0
- package/dist/p2p/unload-model-delegated.js +31 -0
- package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
- package/dist/plugins/builtin/asr-ggml/config.js +46 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
- package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
- package/dist/plugins/builtin/index.d.ts +11 -0
- package/dist/plugins/builtin/index.js +11 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
- package/dist/plugins/index.d.ts +9 -0
- package/dist/plugins/index.js +12 -0
- package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
- package/dist/plugins/ops/bci-transcribe.js +248 -0
- package/dist/plugins/ops/cancel.d.ts +25 -0
- package/dist/plugins/ops/cancel.js +43 -0
- package/dist/plugins/ops/embed.d.ts +15 -0
- package/dist/plugins/ops/embed.js +193 -0
- package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
- package/dist/plugins/ops/kv-cache-retention.js +121 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
- package/dist/plugins/ops/kv-cache-utils.js +120 -0
- package/dist/plugins/ops/load-model.d.ts +8 -0
- package/dist/plugins/ops/load-model.js +82 -0
- package/dist/plugins/ops/transcribe.d.ts +20 -0
- package/dist/plugins/ops/transcribe.js +343 -0
- package/dist/plugins/ops/translate.d.ts +7 -0
- package/dist/plugins/ops/translate.js +256 -0
- package/dist/plugins/ops/unload-model.d.ts +2 -0
- package/dist/plugins/ops/unload-model.js +45 -0
- package/dist/plugins/registry.d.ts +9 -0
- package/dist/plugins/registry.js +107 -0
- package/dist/profiling/aggregator.d.ts +14 -0
- package/dist/profiling/aggregator.js +100 -0
- package/dist/profiling/backend-diagnostics.d.ts +8 -0
- package/dist/profiling/backend-diagnostics.js +36 -0
- package/dist/profiling/clock.d.ts +8 -0
- package/dist/profiling/clock.js +23 -0
- package/dist/profiling/controller.d.ts +34 -0
- package/dist/profiling/controller.js +130 -0
- package/dist/profiling/delegation-profiler.d.ts +31 -0
- package/dist/profiling/delegation-profiler.js +155 -0
- package/dist/profiling/envelope.d.ts +10 -0
- package/dist/profiling/envelope.js +45 -0
- package/dist/profiling/events.d.ts +19 -0
- package/dist/profiling/events.js +35 -0
- package/dist/profiling/exporters.d.ts +9 -0
- package/dist/profiling/exporters.js +242 -0
- package/dist/profiling/index.d.ts +91 -0
- package/dist/profiling/index.js +87 -0
- package/dist/profiling/model-execution.d.ts +15 -0
- package/dist/profiling/model-execution.js +34 -0
- package/dist/profiling/operation-metrics.d.ts +24 -0
- package/dist/profiling/operation-metrics.js +370 -0
- package/dist/profiling/operation-wrappers.d.ts +17 -0
- package/dist/profiling/operation-wrappers.js +196 -0
- package/dist/profiling/profiler.d.ts +27 -0
- package/dist/profiling/profiler.js +127 -0
- package/dist/profiling/ring-buffer.d.ts +18 -0
- package/dist/profiling/ring-buffer.js +83 -0
- package/dist/profiling/types.d.ts +52 -0
- package/dist/profiling/types.js +1 -0
- package/dist/rag/handler.d.ts +2 -0
- package/dist/rag/handler.js +272 -0
- package/dist/rag/handlers/chunk.d.ts +2 -0
- package/dist/rag/handlers/chunk.js +7 -0
- package/dist/rag/handlers/close-workspace.d.ts +2 -0
- package/dist/rag/handlers/close-workspace.js +9 -0
- package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
- package/dist/rag/handlers/delete-embeddings.js +11 -0
- package/dist/rag/handlers/delete-workspace.d.ts +2 -0
- package/dist/rag/handlers/delete-workspace.js +14 -0
- package/dist/rag/handlers/ingest.d.ts +9 -0
- package/dist/rag/handlers/ingest.js +19 -0
- package/dist/rag/handlers/list-workspaces.d.ts +2 -0
- package/dist/rag/handlers/list-workspaces.js +4 -0
- package/dist/rag/handlers/reindex.d.ts +9 -0
- package/dist/rag/handlers/reindex.js +16 -0
- package/dist/rag/handlers/save-embeddings.d.ts +9 -0
- package/dist/rag/handlers/save-embeddings.js +15 -0
- package/dist/rag/handlers/search.d.ts +2 -0
- package/dist/rag/handlers/search.js +16 -0
- package/dist/rag/index.d.ts +11 -0
- package/dist/rag/index.js +11 -0
- package/dist/rag/rag-operation-manager.d.ts +28 -0
- package/dist/rag/rag-operation-manager.js +76 -0
- package/dist/rag/rag-workspace-manager.d.ts +14 -0
- package/dist/rag/rag-workspace-manager.js +143 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +166 -0
- package/dist/resources/collector.d.ts +643 -0
- package/dist/resources/collector.js +226 -0
- package/dist/resources/instance.d.ts +4 -0
- package/dist/resources/instance.js +15 -0
- package/dist/resources/native.d.ts +2 -0
- package/dist/resources/native.js +35 -0
- package/dist/resources/normalize.d.ts +12 -0
- package/dist/resources/normalize.js +73 -0
- package/dist/resources/types.d.ts +65 -0
- package/dist/resources/types.js +1 -0
- package/dist/runtime/cache-lock.d.ts +2 -0
- package/dist/runtime/cache-lock.js +67 -0
- package/dist/runtime/disposable-scope.d.ts +35 -0
- package/dist/runtime/disposable-scope.js +69 -0
- package/dist/runtime/env.d.ts +16 -0
- package/dist/runtime/env.js +53 -0
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +20 -0
- package/dist/runtime/lifecycle.js +102 -0
- package/dist/runtime/logging-stream-registry.d.ts +27 -0
- package/dist/runtime/logging-stream-registry.js +133 -0
- package/dist/runtime/model-config-utils.d.ts +14 -0
- package/dist/runtime/model-config-utils.js +128 -0
- package/dist/runtime/model-registry.d.ts +74 -0
- package/dist/runtime/model-registry.js +130 -0
- package/dist/runtime/registry-client.d.ts +4 -0
- package/dist/runtime/registry-client.js +118 -0
- package/dist/runtime/request-context.d.ts +72 -0
- package/dist/runtime/request-context.js +114 -0
- package/dist/runtime/request-id.d.ts +17 -0
- package/dist/runtime/request-id.js +29 -0
- package/dist/runtime/request-registry.d.ts +194 -0
- package/dist/runtime/request-registry.js +546 -0
- package/dist/runtime/runtime-lifecycle.d.ts +37 -0
- package/dist/runtime/runtime-lifecycle.js +163 -0
- package/dist/runtime/state.d.ts +26 -0
- package/dist/runtime/state.js +138 -0
- package/dist/schemas/audio-gen.d.ts +252 -0
- package/dist/schemas/audio-gen.js +69 -0
- package/dist/schemas/batch-completion-stream.d.ts +467 -0
- package/dist/schemas/batch-completion-stream.js +123 -0
- package/dist/schemas/bci-config.d.ts +74 -0
- package/dist/schemas/bci-config.js +58 -0
- package/dist/schemas/bci.d.ts +178 -0
- package/dist/schemas/bci.js +89 -0
- package/dist/schemas/cancel.d.ts +203 -0
- package/dist/schemas/cancel.js +135 -0
- package/dist/schemas/classification.d.ts +52 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +6420 -0
- package/dist/schemas/common.js +133 -0
- package/dist/schemas/completion-event.d.ts +248 -0
- package/dist/schemas/completion-event.js +94 -0
- package/dist/schemas/completion-stream.d.ts +475 -0
- package/dist/schemas/completion-stream.js +236 -0
- package/dist/schemas/config.d.ts +5022 -0
- package/dist/schemas/config.js +169 -0
- package/dist/schemas/delegate.d.ts +29 -0
- package/dist/schemas/delegate.js +40 -0
- package/dist/schemas/download-asset.d.ts +225 -0
- package/dist/schemas/download-asset.js +71 -0
- package/dist/schemas/embed.d.ts +41 -0
- package/dist/schemas/embed.js +35 -0
- package/dist/schemas/engine-addon-map.d.ts +20 -0
- package/dist/schemas/engine-addon-map.js +78 -0
- package/dist/schemas/error.d.ts +40 -0
- package/dist/schemas/error.js +65 -0
- package/dist/schemas/errors.d.ts +119 -0
- package/dist/schemas/errors.js +615 -0
- package/dist/schemas/finetune.d.ts +621 -0
- package/dist/schemas/finetune.js +163 -0
- package/dist/schemas/index.d.ts +45 -0
- package/dist/schemas/index.js +45 -0
- package/dist/schemas/json-schema.d.ts +15 -0
- package/dist/schemas/json-schema.js +2 -0
- package/dist/schemas/lifecycle.d.ts +38 -0
- package/dist/schemas/lifecycle.js +21 -0
- package/dist/schemas/llamacpp-config.d.ts +403 -0
- package/dist/schemas/llamacpp-config.js +133 -0
- package/dist/schemas/load-model.d.ts +25635 -0
- package/dist/schemas/load-model.js +629 -0
- package/dist/schemas/logging-stream.d.ts +33 -0
- package/dist/schemas/logging-stream.js +17 -0
- package/dist/schemas/mcp-adapter.d.ts +46 -0
- package/dist/schemas/mcp-adapter.js +1 -0
- package/dist/schemas/model-file.d.ts +27 -0
- package/dist/schemas/model-file.js +29 -0
- package/dist/schemas/model-info.d.ts +197 -0
- package/dist/schemas/model-info.js +144 -0
- package/dist/schemas/model-ops.d.ts +36 -0
- package/dist/schemas/model-ops.js +40 -0
- package/dist/schemas/model-src-utils.d.ts +192 -0
- package/dist/schemas/model-src-utils.js +61 -0
- package/dist/schemas/model-types.d.ts +275 -0
- package/dist/schemas/model-types.js +223 -0
- package/dist/schemas/ocr.d.ts +138 -0
- package/dist/schemas/ocr.js +59 -0
- package/dist/schemas/path-validation.d.ts +9 -0
- package/dist/schemas/path-validation.js +22 -0
- package/dist/schemas/plugin.d.ts +341 -0
- package/dist/schemas/plugin.js +241 -0
- package/dist/schemas/profiling.d.ts +695 -0
- package/dist/schemas/profiling.js +100 -0
- package/dist/schemas/provide.d.ts +50 -0
- package/dist/schemas/provide.js +32 -0
- package/dist/schemas/rag.d.ts +301 -0
- package/dist/schemas/rag.js +211 -0
- package/dist/schemas/registry.d.ts +453 -0
- package/dist/schemas/registry.js +104 -0
- package/dist/schemas/reload-config.d.ts +585 -0
- package/dist/schemas/reload-config.js +42 -0
- package/dist/schemas/runtime-context.d.ts +18 -0
- package/dist/schemas/runtime-context.js +7 -0
- package/dist/schemas/sdcpp-config.d.ts +1126 -0
- package/dist/schemas/sdcpp-config.js +748 -0
- package/dist/schemas/system-resources.d.ts +3966 -0
- package/dist/schemas/system-resources.js +149 -0
- package/dist/schemas/text-to-speech.d.ts +1352 -0
- package/dist/schemas/text-to-speech.js +364 -0
- package/dist/schemas/tools.d.ts +80 -0
- package/dist/schemas/tools.js +52 -0
- package/dist/schemas/transcription-config.d.ts +161 -0
- package/dist/schemas/transcription-config.js +156 -0
- package/dist/schemas/transcription.d.ts +261 -0
- package/dist/schemas/transcription.js +145 -0
- package/dist/schemas/translate.d.ts +138 -0
- package/dist/schemas/translate.js +103 -0
- package/dist/schemas/translation-config.d.ts +1031 -0
- package/dist/schemas/translation-config.js +170 -0
- package/dist/schemas/vla.d.ts +110 -0
- package/dist/schemas/vla.js +151 -0
- package/dist/selection.d.ts +8 -0
- package/dist/selection.js +14 -0
- package/dist/surface.d.ts +18 -0
- package/dist/surface.js +30 -0
- package/dist/utils/addon-responses.d.ts +58 -0
- package/dist/utils/addon-responses.js +1 -0
- package/dist/utils/aggregate-events.d.ts +25 -0
- package/dist/utils/aggregate-events.js +73 -0
- package/dist/utils/archive.d.ts +4 -0
- package/dist/utils/archive.js +180 -0
- package/dist/utils/asr-events.d.ts +14 -0
- package/dist/utils/asr-events.js +30 -0
- package/dist/utils/audio/decoder.d.ts +4 -0
- package/dist/utils/audio/decoder.js +80 -0
- package/dist/utils/audio-input.d.ts +9 -0
- package/dist/utils/audio-input.js +34 -0
- package/dist/utils/cache/auto-history.d.ts +3 -0
- package/dist/utils/cache/auto-history.js +26 -0
- package/dist/utils/cache/paths.d.ts +42 -0
- package/dist/utils/cache/paths.js +133 -0
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/cache/types.js +1 -0
- package/dist/utils/cache-normalize.d.ts +9 -0
- package/dist/utils/cache-normalize.js +20 -0
- package/dist/utils/checksum.d.ts +3 -0
- package/dist/utils/checksum.js +29 -0
- package/dist/utils/clear-storage.d.ts +4 -0
- package/dist/utils/clear-storage.js +14 -0
- package/dist/utils/completion-normalizer.d.ts +17 -0
- package/dist/utils/completion-normalizer.js +453 -0
- package/dist/utils/config-transform.d.ts +2 -0
- package/dist/utils/config-transform.js +21 -0
- package/dist/utils/decorate-promise.d.ts +33 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/encoding.d.ts +16 -0
- package/dist/utils/encoding.js +27 -0
- package/dist/utils/expand-gguf-shards.d.ts +2 -0
- package/dist/utils/expand-gguf-shards.js +22 -0
- package/dist/utils/formatting.d.ts +18 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
- package/dist/utils/gguf-tensor-extractor.js +311 -0
- package/dist/utils/global-singleton.d.ts +1 -0
- package/dist/utils/global-singleton.js +10 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/load-model-validation.d.ts +6 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/dist/utils/mcp-adapter.d.ts +11 -0
- package/dist/utils/mcp-adapter.js +96 -0
- package/dist/utils/multi-gpu-mobile.d.ts +4 -0
- package/dist/utils/multi-gpu-mobile.js +10 -0
- package/dist/utils/object.d.ts +1 -0
- package/dist/utils/object.js +3 -0
- package/dist/utils/path-security.d.ts +22 -0
- package/dist/utils/path-security.js +69 -0
- package/dist/utils/qvac-paths.d.ts +1 -0
- package/dist/utils/qvac-paths.js +5 -0
- package/dist/utils/response-format.d.ts +2 -0
- package/dist/utils/response-format.js +20 -0
- package/dist/utils/shard-utils.d.ts +47 -0
- package/dist/utils/shard-utils.js +123 -0
- package/dist/utils/tool-helpers.d.ts +24 -0
- package/dist/utils/tool-helpers.js +111 -0
- package/dist/utils/tool-integration.d.ts +23 -0
- package/dist/utils/tool-integration.js +29 -0
- package/dist/utils/tools/dialect.d.ts +7 -0
- package/dist/utils/tools/dialect.js +20 -0
- package/dist/utils/tools/index.d.ts +2 -0
- package/dist/utils/tools/index.js +2 -0
- package/dist/utils/tools/parser.d.ts +5 -0
- package/dist/utils/tools/parser.js +69 -0
- package/dist/utils/tools/parsers/dsml.d.ts +3 -0
- package/dist/utils/tools/parsers/dsml.js +177 -0
- package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
- package/dist/utils/tools/parsers/gemma4native.js +65 -0
- package/dist/utils/tools/parsers/harmony.d.ts +3 -0
- package/dist/utils/tools/parsers/harmony.js +57 -0
- package/dist/utils/tools/parsers/hermes.d.ts +3 -0
- package/dist/utils/tools/parsers/hermes.js +122 -0
- package/dist/utils/tools/parsers/json.d.ts +5 -0
- package/dist/utils/tools/parsers/json.js +115 -0
- package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
- package/dist/utils/tools/parsers/pythonic.js +267 -0
- package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
- package/dist/utils/tools/parsers/qwen35.js +125 -0
- package/dist/utils/tools/shared.d.ts +17 -0
- package/dist/utils/tools/shared.js +52 -0
- package/dist/utils/transcribe-metadata.d.ts +20 -0
- package/dist/utils/transcribe-metadata.js +28 -0
- package/dist/utils/tts-stats.d.ts +25 -0
- package/dist/utils/tts-stats.js +16 -0
- package/dist/utils/withTimeout.d.ts +17 -0
- package/dist/utils/withTimeout.js +47 -0
- package/dist/utils/zod-error.d.ts +2 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +266 -1
|
@@ -0,0 +1,581 @@
|
|
|
1
|
+
import { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
|
|
4
|
+
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
|
|
5
|
+
import { isMobile } from '../../../../runtime/state.js';
|
|
6
|
+
import { getKVCacheDir } from '../../../../utils/index.js';
|
|
7
|
+
import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
|
|
8
|
+
import { getEngineLogger } from '../../../../logging/index.js';
|
|
9
|
+
// Used by cross-model paths that have no `RequestContext` (e.g.
|
|
10
|
+
// `deleteKvCacheState`). Per-session call sites receive a logger from
|
|
11
|
+
// the caller — typically `withRequestContext(...)`.
|
|
12
|
+
const moduleLogger = getEngineLogger();
|
|
13
|
+
/**
|
|
14
|
+
* Coordinates five KV-cache state layers:
|
|
15
|
+
*
|
|
16
|
+
* 1. `cachedMessageCounts` — saved message boundaries.
|
|
17
|
+
* 2. `initializedCaches` — caches primed in this process.
|
|
18
|
+
* 3. On-disk `.bin` files written by the addon.
|
|
19
|
+
* 4. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
20
|
+
* 5. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
21
|
+
*
|
|
22
|
+
* Every turn must finish through `commitTurn` or `rollback` so all
|
|
23
|
+
* inference state stays aligned, the active-path ref is released, and
|
|
24
|
+
* marker metadata follows the cache directory lifecycle.
|
|
25
|
+
*/
|
|
26
|
+
// ----- module-scoped state. The session is the single mutation point
|
|
27
|
+
// for the in-memory KV-cache bookkeeping. -----
|
|
28
|
+
/**
|
|
29
|
+
* Number of chat messages the kv-cache file on disk is known to cover,
|
|
30
|
+
* keyed by cache path. Written by `commitTurn`, read by `getSavedCount`,
|
|
31
|
+
* deleted by `rollback` / `delete` / `dropStaleSavedCount`. The same
|
|
32
|
+
* INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
|
|
33
|
+
* present only when the corresponding `.bin` file is considered
|
|
34
|
+
* trustworthy. Cancelled or zero-token turns must remove the entry so
|
|
35
|
+
* the next-turn slice doesn't read a stale boundary.
|
|
36
|
+
*/
|
|
37
|
+
const cachedMessageCounts = new Map();
|
|
38
|
+
/**
|
|
39
|
+
* In-memory registry of caches initialized this session. The addon
|
|
40
|
+
* defers disk writes, so the absence of a `.bin` file on disk isn't
|
|
41
|
+
* proof that the cache hasn't been primed in this process. Keyed
|
|
42
|
+
* by `${modelId}:${configHash}:${cacheKey}`, so on-disk caches from
|
|
43
|
+
* older process runs still hit the lazy-load path in `beginTurn`.
|
|
44
|
+
*/
|
|
45
|
+
const initializedCaches = new Set();
|
|
46
|
+
const activeCachePaths = new Map();
|
|
47
|
+
const DESKTOP_AUTO_CACHE_MAX_BYTES = 4 * 1024 * 1024 * 1024;
|
|
48
|
+
const MOBILE_AUTO_CACHE_MAX_BYTES = 512 * 1024 * 1024;
|
|
49
|
+
const AUTO_CACHE_MAX_IDLE_MS = 24 * 60 * 60 * 1000;
|
|
50
|
+
const AUTO_CACHE_SWEEP_INTERVAL_MS = 5 * 60 * 1000;
|
|
51
|
+
let lastAutoCacheSweepMs = 0;
|
|
52
|
+
let autoCacheSweepInFlight = null;
|
|
53
|
+
let cacheStateLockTail = Promise.resolve();
|
|
54
|
+
function initRegistryKey(modelId, configHash, cacheKey) {
|
|
55
|
+
return `${modelId}:${configHash}:${cacheKey}`;
|
|
56
|
+
}
|
|
57
|
+
function markCachePathActive(cachePath) {
|
|
58
|
+
activeCachePaths.set(cachePath, (activeCachePaths.get(cachePath) ?? 0) + 1);
|
|
59
|
+
}
|
|
60
|
+
function releaseCachePath(cachePath) {
|
|
61
|
+
const count = activeCachePaths.get(cachePath);
|
|
62
|
+
if (count === undefined)
|
|
63
|
+
return;
|
|
64
|
+
if (count === 1) {
|
|
65
|
+
activeCachePaths.delete(cachePath);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
activeCachePaths.set(cachePath, count - 1);
|
|
69
|
+
}
|
|
70
|
+
function isCacheKeyActive(cacheKey) {
|
|
71
|
+
const cacheDirectory = path.join(getKVCacheDir(), cacheKey);
|
|
72
|
+
return Array.from(activeCachePaths.keys()).some((cachePath) => isCachePathWithinDirectory(cacheDirectory, cachePath));
|
|
73
|
+
}
|
|
74
|
+
function getAutoCacheMaxBytes() {
|
|
75
|
+
return isMobile() ? MOBILE_AUTO_CACHE_MAX_BYTES : DESKTOP_AUTO_CACHE_MAX_BYTES;
|
|
76
|
+
}
|
|
77
|
+
async function withCacheStateLock(operation) {
|
|
78
|
+
const previous = cacheStateLockTail;
|
|
79
|
+
let releaseLock = () => { };
|
|
80
|
+
cacheStateLockTail = new Promise((resolve) => {
|
|
81
|
+
releaseLock = resolve;
|
|
82
|
+
});
|
|
83
|
+
await previous;
|
|
84
|
+
try {
|
|
85
|
+
return await operation();
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
releaseLock();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
async function maybeSweepAutoCaches(logger, overrides) {
|
|
92
|
+
const nowMs = overrides?.nowMs ?? Date.now();
|
|
93
|
+
if (autoCacheSweepInFlight !== null) {
|
|
94
|
+
await autoCacheSweepInFlight;
|
|
95
|
+
if (!overrides?.force)
|
|
96
|
+
return;
|
|
97
|
+
return maybeSweepAutoCaches(logger, overrides);
|
|
98
|
+
}
|
|
99
|
+
if (!overrides?.force && nowMs - lastAutoCacheSweepMs < AUTO_CACHE_SWEEP_INTERVAL_MS)
|
|
100
|
+
return;
|
|
101
|
+
lastAutoCacheSweepMs = nowMs;
|
|
102
|
+
const sweep = async () => {
|
|
103
|
+
try {
|
|
104
|
+
const cacheKeys = await planAutoCacheEvictions({
|
|
105
|
+
activeCachePaths: Array.from(activeCachePaths.keys()),
|
|
106
|
+
maxBytes: overrides?.maxBytes ?? getAutoCacheMaxBytes(),
|
|
107
|
+
maxIdleMs: overrides?.maxIdleMs ?? AUTO_CACHE_MAX_IDLE_MS,
|
|
108
|
+
nowMs
|
|
109
|
+
});
|
|
110
|
+
let evictionCount = 0;
|
|
111
|
+
await withCacheStateLock(async () => {
|
|
112
|
+
for (const cacheKey of cacheKeys) {
|
|
113
|
+
if (isCacheKeyActive(cacheKey))
|
|
114
|
+
continue;
|
|
115
|
+
await deleteKvCacheState({ kvCacheKey: cacheKey });
|
|
116
|
+
evictionCount++;
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
if (evictionCount > 0) {
|
|
120
|
+
logger.debug(`[kv-cache] Evicted ${evictionCount} inactive auto-cache entries`);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
logger.warn(`[kv-cache] Auto-cache retention sweep failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
const sweepPromise = sweep();
|
|
128
|
+
autoCacheSweepInFlight = sweepPromise;
|
|
129
|
+
try {
|
|
130
|
+
await sweepPromise;
|
|
131
|
+
}
|
|
132
|
+
finally {
|
|
133
|
+
autoCacheSweepInFlight = null;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
function scheduleAutoCacheSweep(logger) {
|
|
137
|
+
void maybeSweepAutoCaches(logger).catch((error) => {
|
|
138
|
+
logger.warn(`[kv-cache] Failed to schedule auto-cache retention sweep: ${error instanceof Error ? error.message : String(error)}`);
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
// ----- factory -----
|
|
142
|
+
/**
|
|
143
|
+
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
144
|
+
* scope. `options.logger` is the per-instance logger the session emits
|
|
145
|
+
* through (typically `withRequestContext(getEngineLogger(), ctx)`);
|
|
146
|
+
* falls back to the module-scoped logger when omitted.
|
|
147
|
+
*/
|
|
148
|
+
export function createKvCacheSession(modelId, options) {
|
|
149
|
+
const logger = options?.logger ?? moduleLogger;
|
|
150
|
+
// Per-session map: each `TurnHandle` carries an opaque entry here. A
|
|
151
|
+
// WeakMap so handles drop their state once the handler scope releases
|
|
152
|
+
// the reference; the module-scoped maps above survive.
|
|
153
|
+
const turnState = new WeakMap();
|
|
154
|
+
function makeHandle(cachePath, registryKey, autoCacheKey) {
|
|
155
|
+
const handle = {
|
|
156
|
+
cachePath,
|
|
157
|
+
savedCount: cachedMessageCounts.get(cachePath) ?? 0
|
|
158
|
+
};
|
|
159
|
+
turnState.set(handle, {
|
|
160
|
+
cachePath,
|
|
161
|
+
registryKey,
|
|
162
|
+
...(autoCacheKey !== undefined && { autoCacheKey }),
|
|
163
|
+
committed: false,
|
|
164
|
+
rolledBack: false
|
|
165
|
+
});
|
|
166
|
+
markCachePathActive(cachePath);
|
|
167
|
+
return handle;
|
|
168
|
+
}
|
|
169
|
+
async function beginCustom(input) {
|
|
170
|
+
const cachePath = await getCacheFilePath(modelId, input.configHash, input.customKey);
|
|
171
|
+
const registryKey = initRegistryKey(modelId, input.configHash, input.customKey);
|
|
172
|
+
const handle = makeHandle(cachePath, registryKey);
|
|
173
|
+
try {
|
|
174
|
+
// In-memory registry check first — the addon defers disk writes, so
|
|
175
|
+
// a freshly-primed cache may not yet exist on disk. If the
|
|
176
|
+
// in-memory flag isn't set, fall back to a filesystem probe so
|
|
177
|
+
// caches surviving across process restarts still hit the reuse path.
|
|
178
|
+
let exists = initializedCaches.has(registryKey);
|
|
179
|
+
if (!exists) {
|
|
180
|
+
try {
|
|
181
|
+
await fsPromises.access(cachePath);
|
|
182
|
+
exists = true;
|
|
183
|
+
initializedCaches.add(registryKey);
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
exists = false;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
logCacheStatus(input.customKey, exists);
|
|
190
|
+
if (!exists) {
|
|
191
|
+
await input.primeIfMissing(cachePath);
|
|
192
|
+
await verifyPrimedFile(cachePath, logger);
|
|
193
|
+
initializedCaches.add(registryKey);
|
|
194
|
+
}
|
|
195
|
+
return handle;
|
|
196
|
+
}
|
|
197
|
+
catch (error) {
|
|
198
|
+
releaseCachePath(cachePath);
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
async function beginAuto(input) {
|
|
203
|
+
// The pre-response cache key is derived from
|
|
204
|
+
// `history.slice(0, -1)` — `findMatchingCache` does that
|
|
205
|
+
// internally. The post-response key (used after a successful turn)
|
|
206
|
+
// is computed by the caller and passed to `commitTurn` as
|
|
207
|
+
// `targetCachePath`.
|
|
208
|
+
const setup = await withCacheStateLock(async () => {
|
|
209
|
+
const existingCache = await findMatchingCache(modelId, input.configHash, input.history);
|
|
210
|
+
const cacheInfo = existingCache ?? (await getCurrentCacheInfo(modelId, input.configHash, input.history));
|
|
211
|
+
const registryKey = initRegistryKey(modelId, input.configHash, cacheInfo.cacheKey);
|
|
212
|
+
return {
|
|
213
|
+
cacheExists: existingCache !== null,
|
|
214
|
+
cachePath: cacheInfo.cachePath,
|
|
215
|
+
cacheKey: cacheInfo.cacheKey,
|
|
216
|
+
registryKey,
|
|
217
|
+
handle: makeHandle(cacheInfo.cachePath, registryKey, cacheInfo.cacheKey)
|
|
218
|
+
};
|
|
219
|
+
});
|
|
220
|
+
const { cacheExists, cachePath, cacheKey, registryKey, handle } = setup;
|
|
221
|
+
logCacheStatus('auto', cacheExists);
|
|
222
|
+
try {
|
|
223
|
+
if (!cacheExists) {
|
|
224
|
+
await input.primeIfMissing(cachePath);
|
|
225
|
+
await verifyPrimedFile(cachePath, logger);
|
|
226
|
+
initializedCaches.add(registryKey);
|
|
227
|
+
}
|
|
228
|
+
return handle;
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
releaseCachePath(cachePath);
|
|
232
|
+
await pruneEmptyCacheDirectories(cachePath);
|
|
233
|
+
await removeAutoCacheMarkerIfMissing(cacheKey);
|
|
234
|
+
throw error;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
async function beginTurn(input) {
|
|
238
|
+
if (input.kind === 'custom')
|
|
239
|
+
return beginCustom(input);
|
|
240
|
+
return beginAuto(input);
|
|
241
|
+
}
|
|
242
|
+
async function commitTurn(turn, result) {
|
|
243
|
+
const state = turnState.get(turn);
|
|
244
|
+
if (!state) {
|
|
245
|
+
// Handle from a different session or already torn down. Treat as
|
|
246
|
+
// no-op — caller shouldn't be reaching into another session's
|
|
247
|
+
// state, but failing loudly here punishes the rollback-after-end
|
|
248
|
+
// path more than it helps.
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
if (state.committed || state.rolledBack)
|
|
252
|
+
return;
|
|
253
|
+
if (result.kind === 'static') {
|
|
254
|
+
// Custom-key path: the addon wrote the new cache state inline
|
|
255
|
+
// at the same path. Verify the file persisted (the addon
|
|
256
|
+
// currently swallows save errors — see TODO in
|
|
257
|
+
// `verifySaveAndRecord`) and record the new boundary.
|
|
258
|
+
const ok = await verifySaveAndRecord(state.cachePath, result.messageCount);
|
|
259
|
+
if (!ok) {
|
|
260
|
+
// The expected save didn't land — treat the turn as a rollback
|
|
261
|
+
// so the next turn re-primes cleanly.
|
|
262
|
+
await runRollback(state);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
state.committed = true;
|
|
266
|
+
releaseCachePath(state.cachePath);
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const sourceCachePath = state.cachePath;
|
|
270
|
+
const sourceCacheKey = state.autoCacheKey;
|
|
271
|
+
const targetCacheKey = path.basename(path.dirname(path.dirname(result.targetCachePath)));
|
|
272
|
+
const renamed = await withCacheStateLock(async () => {
|
|
273
|
+
markCachePathActive(result.targetCachePath);
|
|
274
|
+
await fsPromises.mkdir(path.dirname(result.targetCachePath), { recursive: true });
|
|
275
|
+
await markAutoCacheKey(targetCacheKey);
|
|
276
|
+
if (!(await renameCacheFile(sourceCachePath, result.targetCachePath))) {
|
|
277
|
+
releaseCachePath(result.targetCachePath);
|
|
278
|
+
await pruneEmptyCacheDirectories(result.targetCachePath);
|
|
279
|
+
await removeAutoCacheMarkerIfMissing(targetCacheKey);
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
releaseCachePath(sourceCachePath);
|
|
283
|
+
state.cachePath = result.targetCachePath;
|
|
284
|
+
state.autoCacheKey = targetCacheKey;
|
|
285
|
+
await pruneEmptyCacheDirectories(sourceCachePath);
|
|
286
|
+
if (sourceCacheKey !== undefined) {
|
|
287
|
+
await removeAutoCacheMarkerIfMissing(sourceCacheKey);
|
|
288
|
+
}
|
|
289
|
+
cachedMessageCounts.delete(sourceCachePath);
|
|
290
|
+
initializedCaches.delete(state.registryKey);
|
|
291
|
+
return true;
|
|
292
|
+
});
|
|
293
|
+
if (!renamed) {
|
|
294
|
+
logger.warn(`[kv-cache] Auto cache rename failed; rolling back. from=${sourceCachePath} to=${result.targetCachePath}`);
|
|
295
|
+
await runRollback(state);
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
const ok = await verifySaveAndRecord(result.targetCachePath, result.messageCount);
|
|
299
|
+
if (!ok) {
|
|
300
|
+
// Rename succeeded but the file isn't where we expected. Roll
|
|
301
|
+
// back via the target path instead of the (now-empty) source.
|
|
302
|
+
await runRollback(state);
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
// Successful auto-rename. The handle's `cachePath` field still
|
|
306
|
+
// points at the (now-gone) source path — that's fine, the handle
|
|
307
|
+
// is committed and won't roll back. Future turns compute fresh
|
|
308
|
+
// paths.
|
|
309
|
+
state.committed = true;
|
|
310
|
+
releaseCachePath(state.cachePath);
|
|
311
|
+
scheduleAutoCacheSweep(logger);
|
|
312
|
+
}
|
|
313
|
+
async function rollback(turn) {
|
|
314
|
+
const state = turnState.get(turn);
|
|
315
|
+
if (!state)
|
|
316
|
+
return;
|
|
317
|
+
if (state.committed || state.rolledBack)
|
|
318
|
+
return;
|
|
319
|
+
await runRollback(state);
|
|
320
|
+
}
|
|
321
|
+
async function runRollback(state) {
|
|
322
|
+
// Order matters only weakly: unlink first so a partial disk-state
|
|
323
|
+
// can't be re-loaded by a sibling turn between the file delete and
|
|
324
|
+
// the in-memory clear. In practice handlers serialise per model;
|
|
325
|
+
// the order is belt-and-suspenders.
|
|
326
|
+
try {
|
|
327
|
+
await fsPromises.unlink(state.cachePath);
|
|
328
|
+
}
|
|
329
|
+
catch (unlinkError) {
|
|
330
|
+
logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
|
|
331
|
+
}
|
|
332
|
+
await pruneEmptyCacheDirectories(state.cachePath);
|
|
333
|
+
if (state.autoCacheKey !== undefined) {
|
|
334
|
+
await removeAutoCacheMarkerIfMissing(state.autoCacheKey);
|
|
335
|
+
}
|
|
336
|
+
initializedCaches.delete(state.registryKey);
|
|
337
|
+
cachedMessageCounts.delete(state.cachePath);
|
|
338
|
+
releaseCachePath(state.cachePath);
|
|
339
|
+
state.rolledBack = true;
|
|
340
|
+
if (state.autoCacheKey !== undefined)
|
|
341
|
+
scheduleAutoCacheSweep(logger);
|
|
342
|
+
}
|
|
343
|
+
function dropStaleSavedCount(turn) {
|
|
344
|
+
const state = turnState.get(turn);
|
|
345
|
+
if (!state)
|
|
346
|
+
return;
|
|
347
|
+
cachedMessageCounts.delete(state.cachePath);
|
|
348
|
+
}
|
|
349
|
+
return {
|
|
350
|
+
beginTurn,
|
|
351
|
+
commitTurn,
|
|
352
|
+
rollback,
|
|
353
|
+
dropStaleSavedCount
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
// ----- module-level administrative API -----
|
|
357
|
+
/**
|
|
358
|
+
* Atomically delete every layer of KV-cache state for a
|
|
359
|
+
* `(kvCacheKey, modelId)` pair, or wipe everything. Single entry point
|
|
360
|
+
* — the only mutation point for cross-model state outside of
|
|
361
|
+
* turn-scoped `commitTurn`/`rollback`.
|
|
362
|
+
*
|
|
363
|
+
* Why this isn't a method on `KvCacheSession`: deletes are
|
|
364
|
+
* cross-model (`all: true` has no model; the keyed form has
|
|
365
|
+
* `modelId` optional in the request). A session, by contrast, is
|
|
366
|
+
* created with a *fixed* `modelId` for the duration of a turn. Making
|
|
367
|
+
* delete a method would force callers to materialise an irrelevant
|
|
368
|
+
* session for cross-model administrative cleanups.
|
|
369
|
+
*
|
|
370
|
+
* Layers cleared, in order:
|
|
371
|
+
* 1. On-disk: `deleteCache(...)` removes the matching directory
|
|
372
|
+
* tree (or wipes and recreates the root for `all: true`).
|
|
373
|
+
* 2. `cachedMessageCounts`: prefix-cleanup by the removed directory
|
|
374
|
+
* so any per-cache count under the deleted tree is forgotten.
|
|
375
|
+
* 3. `initializedCaches`: scope clear by `(kvCacheKey[, modelId])`,
|
|
376
|
+
* matching the on-disk scope.
|
|
377
|
+
*
|
|
378
|
+
* Concurrency with in-flight turns: this delete is wire-async with
|
|
379
|
+
* respect to any turn currently holding a `TurnHandle` for the same
|
|
380
|
+
* cache key. Worst case the on-disk `.bin` is removed while a turn is
|
|
381
|
+
* mid-write; the turn's eventual `commitTurn(...)` then fails the
|
|
382
|
+
* `verifySaveAndRecord` probe (file gone) and rolls back idempotently.
|
|
383
|
+
* No coordination primitive is needed because every layer's mutation
|
|
384
|
+
* is idempotent (`unlink` no-ops if missing, `Map.delete` / `Set.delete`
|
|
385
|
+
* no-op on absent keys).
|
|
386
|
+
*/
|
|
387
|
+
export async function deleteKvCacheState(target) {
|
|
388
|
+
if ('all' in target) {
|
|
389
|
+
const removed = await deleteCacheUtil({ all: true });
|
|
390
|
+
cachedMessageCounts.clear();
|
|
391
|
+
initializedCaches.clear();
|
|
392
|
+
// `removed` is the kv-cache root dir; surfaces it for ops
|
|
393
|
+
// visibility but isn't part of the contract.
|
|
394
|
+
moduleLogger.debug(`[kv-cache] Cleared all caches under ${removed}`);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
const removedPath = await deleteCacheUtil({
|
|
398
|
+
kvCacheKey: target.kvCacheKey,
|
|
399
|
+
...(target.modelId !== undefined && { modelId: target.modelId })
|
|
400
|
+
});
|
|
401
|
+
if (target.modelId === undefined) {
|
|
402
|
+
await removeAutoCacheMarker(target.kvCacheKey);
|
|
403
|
+
}
|
|
404
|
+
// Prefix-cleanup the in-memory counts. The on-disk directory tree
|
|
405
|
+
// is `{kvCacheRoot}/{kvCacheKey}[/{modelId}]/`, so every entry in
|
|
406
|
+
// `cachedMessageCounts` whose key is the removed directory itself
|
|
407
|
+
// or sits beneath it must go.
|
|
408
|
+
clearCachedMessageCountsByPrefix(removedPath, path.sep);
|
|
409
|
+
// The in-memory init-set keys are
|
|
410
|
+
// `${modelId}:${configHash}:${kvCacheKey}` — clear by the user-
|
|
411
|
+
// facing kvCacheKey (and optionally narrow by modelId).
|
|
412
|
+
clearInitializedCachesByScope({
|
|
413
|
+
cacheKey: target.kvCacheKey,
|
|
414
|
+
...(target.modelId !== undefined && { modelId: target.modelId })
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
// ----- private helpers -----
|
|
418
|
+
/**
|
|
419
|
+
* Verify that the addon actually persisted a usable cache file after a
|
|
420
|
+
* prime. Mirrors the `verifySaveAndRecord` access-probe used at commit
|
|
421
|
+
* time, applied at prime time so the session doesn't mark a cache
|
|
422
|
+
* `initializedCaches.add(...)` against a path that's missing or empty
|
|
423
|
+
* on disk.
|
|
424
|
+
*
|
|
425
|
+
* Failure modes this catches:
|
|
426
|
+
*
|
|
427
|
+
* - The addon's `model.run({ saveSessionPath })` was interrupted
|
|
428
|
+
* before the save call ran (e.g. signal abort during prefill); the
|
|
429
|
+
* prime closure resolves cleanly because addon save errors are not
|
|
430
|
+
* propagated, but no file is on disk.
|
|
431
|
+
* - The addon's `llama_state_save_file` was called but produced an
|
|
432
|
+
* empty file (out-of-space / fs error swallowed by the addon).
|
|
433
|
+
*
|
|
434
|
+
* Failure modes this does **NOT** catch:
|
|
435
|
+
*
|
|
436
|
+
* - A partial-but-nonzero file written by the addon (e.g. header +
|
|
437
|
+
* truncated KV state). Catching this requires either an
|
|
438
|
+
* addon-side change (have `CacheManager::writeCacheFile` check the
|
|
439
|
+
* return value of `llama_state_save_file` and throw on failure) or
|
|
440
|
+
* a structural hash check we can't currently compute
|
|
441
|
+
* engine-side. Filed as a follow-up — see `cache-api.md` in the addon
|
|
442
|
+
* repo / tracking ticket.
|
|
443
|
+
*
|
|
444
|
+
* On failure we best-effort `unlink` an empty leftover file (so the
|
|
445
|
+
* next existence probe doesn't trust it) and throw — the handler in
|
|
446
|
+
* `completion-stream.ts` lets the error propagate up and no
|
|
447
|
+
* `initializedCaches` entry is recorded.
|
|
448
|
+
*/
|
|
449
|
+
async function verifyPrimedFile(cachePath, logger) {
|
|
450
|
+
let stats;
|
|
451
|
+
try {
|
|
452
|
+
stats = await fsPromises.stat(cachePath);
|
|
453
|
+
}
|
|
454
|
+
catch (statError) {
|
|
455
|
+
// ENOENT is the common case here — addon prime returned without
|
|
456
|
+
// calling save (most often: signal abort during prefill).
|
|
457
|
+
await pruneEmptyCacheDirectories(cachePath);
|
|
458
|
+
throw new Error(`[kv-cache] prime closure resolved but no cache file was written. path=${cachePath} cause=${statError instanceof Error ? statError.message : String(statError)}`);
|
|
459
|
+
}
|
|
460
|
+
if (stats.size === 0) {
|
|
461
|
+
// Best-effort cleanup so a future probe doesn't trust the empty
|
|
462
|
+
// file. Unlink failure is non-fatal — we still throw on the
|
|
463
|
+
// primary "prime didn't persist" condition.
|
|
464
|
+
try {
|
|
465
|
+
await fsPromises.unlink(cachePath);
|
|
466
|
+
await pruneEmptyCacheDirectories(cachePath);
|
|
467
|
+
}
|
|
468
|
+
catch (unlinkError) {
|
|
469
|
+
logger.warn(`[kv-cache] Failed to remove empty primed cache file. path=${cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
|
|
470
|
+
}
|
|
471
|
+
throw new Error(`[kv-cache] prime closure resolved but cache file is empty. path=${cachePath}`);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
/**
|
|
475
|
+
* Verify the addon actually persisted the cache file before recording
|
|
476
|
+
* its message count. The addon currently swallows write errors
|
|
477
|
+
* silently, so a missing file means the next turn must resend the full
|
|
478
|
+
* history rather than slicing against a stale `savedCount`.
|
|
479
|
+
*
|
|
480
|
+
* TODO: once the addon surfaces save failures (e.g. throws
|
|
481
|
+
* `UnableToSaveSessionFile` when `llama_state_save_file` returns
|
|
482
|
+
* false), drop the `access()` probe and wrap the `model.run()` call in
|
|
483
|
+
* a real try/catch that forwards the error.
|
|
484
|
+
*/
|
|
485
|
+
async function verifySaveAndRecord(cachePath, messageCount) {
|
|
486
|
+
try {
|
|
487
|
+
await fsPromises.access(cachePath);
|
|
488
|
+
cachedMessageCounts.set(cachePath, messageCount);
|
|
489
|
+
return true;
|
|
490
|
+
}
|
|
491
|
+
catch (err) {
|
|
492
|
+
cachedMessageCounts.delete(cachePath);
|
|
493
|
+
logCacheSaveError(cachePath, err);
|
|
494
|
+
return false;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
function clearCachedMessageCountsByPrefix(prefix, sep) {
|
|
498
|
+
if (!prefix) {
|
|
499
|
+
cachedMessageCounts.clear();
|
|
500
|
+
return;
|
|
501
|
+
}
|
|
502
|
+
for (const key of cachedMessageCounts.keys()) {
|
|
503
|
+
if (key === prefix) {
|
|
504
|
+
cachedMessageCounts.delete(key);
|
|
505
|
+
continue;
|
|
506
|
+
}
|
|
507
|
+
if (!key.startsWith(prefix + sep))
|
|
508
|
+
continue;
|
|
509
|
+
cachedMessageCounts.delete(key);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
function clearInitializedCachesByScope(scope) {
|
|
513
|
+
if (scope.cacheKey === undefined && scope.modelId === undefined) {
|
|
514
|
+
initializedCaches.clear();
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
for (const key of initializedCaches) {
|
|
518
|
+
const firstSep = key.indexOf(':');
|
|
519
|
+
const secondSep = key.indexOf(':', firstSep + 1);
|
|
520
|
+
if (firstSep === -1 || secondSep === -1)
|
|
521
|
+
continue;
|
|
522
|
+
const entryModelId = key.slice(0, firstSep);
|
|
523
|
+
const entryCacheKey = key.slice(secondSep + 1);
|
|
524
|
+
if (scope.cacheKey !== undefined && entryCacheKey !== scope.cacheKey) {
|
|
525
|
+
continue;
|
|
526
|
+
}
|
|
527
|
+
if (scope.modelId !== undefined && entryModelId !== scope.modelId) {
|
|
528
|
+
continue;
|
|
529
|
+
}
|
|
530
|
+
initializedCaches.delete(key);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
// ----- test-only escape hatches -----
|
|
534
|
+
/**
|
|
535
|
+
* Test-only access to the module-scoped state. Production code reaches
|
|
536
|
+
* for cache state exclusively through the session API; the unit suite
|
|
537
|
+
* for `kv-cache-session.test.ts` needs to seed and inspect raw state
|
|
538
|
+
* to assert the rollback / commit invariants. Not part of the public
|
|
539
|
+
* surface.
|
|
540
|
+
*
|
|
541
|
+
* @internal
|
|
542
|
+
*/
|
|
543
|
+
export const __kvCacheSessionTestHooks = {
|
|
544
|
+
getSavedCount(cachePath) {
|
|
545
|
+
return cachedMessageCounts.get(cachePath);
|
|
546
|
+
},
|
|
547
|
+
setSavedCountForTest(cachePath, count) {
|
|
548
|
+
cachedMessageCounts.set(cachePath, count);
|
|
549
|
+
},
|
|
550
|
+
hasInitializedKey(modelId, configHash, cacheKey) {
|
|
551
|
+
return initializedCaches.has(initRegistryKey(modelId, configHash, cacheKey));
|
|
552
|
+
},
|
|
553
|
+
markInitializedForTest(modelId, configHash, cacheKey) {
|
|
554
|
+
initializedCaches.add(initRegistryKey(modelId, configHash, cacheKey));
|
|
555
|
+
},
|
|
556
|
+
getLastAutoCacheSweepMsForTest() {
|
|
557
|
+
return lastAutoCacheSweepMs;
|
|
558
|
+
},
|
|
559
|
+
setLastAutoCacheSweepMsForTest(value) {
|
|
560
|
+
lastAutoCacheSweepMs = value;
|
|
561
|
+
},
|
|
562
|
+
resetForTest() {
|
|
563
|
+
cachedMessageCounts.clear();
|
|
564
|
+
initializedCaches.clear();
|
|
565
|
+
activeCachePaths.clear();
|
|
566
|
+
lastAutoCacheSweepMs = 0;
|
|
567
|
+
autoCacheSweepInFlight = null;
|
|
568
|
+
cacheStateLockTail = Promise.resolve();
|
|
569
|
+
},
|
|
570
|
+
waitForAutoCacheSweepForTest() {
|
|
571
|
+
return autoCacheSweepInFlight ?? Promise.resolve();
|
|
572
|
+
},
|
|
573
|
+
sweepAutoCachesForTest(options) {
|
|
574
|
+
return maybeSweepAutoCaches(moduleLogger, { ...options, force: true });
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
// Re-export `generateConfigHash` from the path utilities so callers of
|
|
578
|
+
// the session can compute the hash without separately importing
|
|
579
|
+
// `kv-cache-utils`. The function itself stays in `kv-cache-utils.ts`
|
|
580
|
+
// (pure, no state) — only the re-export lives here.
|
|
581
|
+
export { generateConfigHash };
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure decision helper for kv-cache history slicing used by
|
|
3
|
+
* `completion-stream.ts` (via the slice computed from
|
|
4
|
+
* `TurnHandle.savedCount`).
|
|
5
|
+
*
|
|
6
|
+
* This module intentionally has **no** `bare-*` imports so it can be
|
|
7
|
+
* exercised directly from unit tests running under `bun` without
|
|
8
|
+
* pulling in the Bare runtime (which is not available in that
|
|
9
|
+
* environment).
|
|
10
|
+
*
|
|
11
|
+
* Cancel detection flows through the per-request `AbortSignal` from
|
|
12
|
+
* `RequestRegistry`; the `cachedMessageCounts` map lives in
|
|
13
|
+
* `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
|
|
14
|
+
* layers (saved counts, init flags, on-disk files). Only the pure
|
|
15
|
+
* slice-decision helper remains here.
|
|
16
|
+
*/
|
|
17
|
+
export interface HistoryMessage {
|
|
18
|
+
role: string;
|
|
19
|
+
content: string;
|
|
20
|
+
attachments?: {
|
|
21
|
+
path: string;
|
|
22
|
+
}[] | undefined;
|
|
23
|
+
}
|
|
24
|
+
export interface HistorySliceDecision {
|
|
25
|
+
/** Messages to send to the model on the next turn. */
|
|
26
|
+
messages: HistoryMessage[];
|
|
27
|
+
/**
|
|
28
|
+
* True when the decision path proves the current `savedCount` is stale
|
|
29
|
+
* and the caller should drop the cached entry (via
|
|
30
|
+
* `KvCacheSession.dropStaleSavedCount(turn)`) to avoid propagating the
|
|
31
|
+
* bad count to the next turn.
|
|
32
|
+
*/
|
|
33
|
+
clearStaleCount: boolean;
|
|
34
|
+
}
|
|
35
|
+
interface CacheCommitContext {
|
|
36
|
+
aborted: boolean;
|
|
37
|
+
producedTokens: boolean;
|
|
38
|
+
generatedTokens?: number | undefined;
|
|
39
|
+
predict?: number | undefined;
|
|
40
|
+
}
|
|
41
|
+
export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Pure slice decision for `prepareMessagesForCache`.
|
|
44
|
+
*
|
|
45
|
+
* Mirrors the shape of the logic in `completion-stream.ts` but without
|
|
46
|
+
* calling `transformMessages` (which depends on `bare-fs` for
|
|
47
|
+
* attachment probing). Kept here so the decision can be unit-tested in
|
|
48
|
+
* isolation.
|
|
49
|
+
*
|
|
50
|
+
* The key regression guard: when a non-zero `savedCount` would slice
|
|
51
|
+
* the history down to an empty array, it is treated as stale — the
|
|
52
|
+
* caller falls back to sending the system-stripped full history rather
|
|
53
|
+
* than handing the model an empty payload.
|
|
54
|
+
*/
|
|
55
|
+
export declare function decideCachedHistorySlice(savedCount: number, cacheExists: boolean, history: HistoryMessage[]): HistorySliceDecision;
|
|
56
|
+
export {};
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure decision helper for kv-cache history slicing used by
|
|
3
|
+
* `completion-stream.ts` (via the slice computed from
|
|
4
|
+
* `TurnHandle.savedCount`).
|
|
5
|
+
*
|
|
6
|
+
* This module intentionally has **no** `bare-*` imports so it can be
|
|
7
|
+
* exercised directly from unit tests running under `bun` without
|
|
8
|
+
* pulling in the Bare runtime (which is not available in that
|
|
9
|
+
* environment).
|
|
10
|
+
*
|
|
11
|
+
* Cancel detection flows through the per-request `AbortSignal` from
|
|
12
|
+
* `RequestRegistry`; the `cachedMessageCounts` map lives in
|
|
13
|
+
* `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
|
|
14
|
+
* layers (saved counts, init flags, on-disk files). Only the pure
|
|
15
|
+
* slice-decision helper remains here.
|
|
16
|
+
*/
|
|
17
|
+
export function shouldCommitCachedTurn(context) {
|
|
18
|
+
const { aborted, producedTokens, generatedTokens, predict } = context;
|
|
19
|
+
const stoppedByBudget = predict !== undefined &&
|
|
20
|
+
predict > 0 &&
|
|
21
|
+
generatedTokens !== undefined &&
|
|
22
|
+
generatedTokens >= predict;
|
|
23
|
+
return !aborted && producedTokens && !stoppedByBudget;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Pure slice decision for `prepareMessagesForCache`.
|
|
27
|
+
*
|
|
28
|
+
* Mirrors the shape of the logic in `completion-stream.ts` but without
|
|
29
|
+
* calling `transformMessages` (which depends on `bare-fs` for
|
|
30
|
+
* attachment probing). Kept here so the decision can be unit-tested in
|
|
31
|
+
* isolation.
|
|
32
|
+
*
|
|
33
|
+
* The key regression guard: when a non-zero `savedCount` would slice
|
|
34
|
+
* the history down to an empty array, it is treated as stale — the
|
|
35
|
+
* caller falls back to sending the system-stripped full history rather
|
|
36
|
+
* than handing the model an empty payload.
|
|
37
|
+
*/
|
|
38
|
+
export function decideCachedHistorySlice(savedCount, cacheExists, history) {
|
|
39
|
+
if (!cacheExists || history.length === 0) {
|
|
40
|
+
return {
|
|
41
|
+
messages: history.filter((msg) => msg.role !== 'system'),
|
|
42
|
+
clearStaleCount: false
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
const canSlice = savedCount > 0 && savedCount <= history.length;
|
|
46
|
+
const sliced = canSlice ? history.slice(savedCount) : null;
|
|
47
|
+
// A non-null slice that is empty means the saved count is stale: the
|
|
48
|
+
// cached turn boundary is claiming the entire current history is
|
|
49
|
+
// already cached, which happens when a previous turn was cancelled
|
|
50
|
+
// mid-decode and still recorded `history.length + 1`. Treat it as a
|
|
51
|
+
// bad state and resend the full (system-stripped) history.
|
|
52
|
+
const useSlice = sliced !== null && sliced.length > 0;
|
|
53
|
+
const messages = useSlice ? sliced : history.filter((msg) => msg.role !== 'system');
|
|
54
|
+
return {
|
|
55
|
+
messages,
|
|
56
|
+
clearStaleCount: !useSlice && savedCount > 0
|
|
57
|
+
};
|
|
58
|
+
}
|