@qvac/inference 0.18.2 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +69 -36
- package/README.md +1 -5
- package/dist/api/assess-model-fit.d.ts +16 -0
- package/dist/api/assess-model-fit.js +27 -0
- package/dist/api/audio-gen.d.ts +4 -4
- package/dist/api/audio-gen.js +17 -6
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/download-asset.d.ts +1 -1
- package/dist/api/download-asset.js +2 -2
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-loaded-model-info.d.ts +6 -10
- package/dist/api/get-loaded-model-info.js +6 -10
- package/dist/api/heartbeat.d.ts +5 -22
- package/dist/api/heartbeat.js +6 -23
- package/dist/api/index.d.ts +36 -35
- package/dist/api/index.js +35 -35
- package/dist/api/load-model.d.ts +5 -1
- package/dist/api/load-model.js +3 -5
- package/dist/api/logging-stream-registry.js +1 -1
- package/dist/api/resume.d.ts +0 -3
- package/dist/api/resume.js +0 -3
- package/dist/api/subscribe-logs.js +1 -1
- package/dist/api/suspend.d.ts +0 -4
- package/dist/api/suspend.js +0 -4
- package/dist/api/translate.d.ts +3 -1
- package/dist/api/translate.js +16 -8
- package/dist/api/unload-model.js +3 -5
- package/dist/api/video-request.d.ts +207 -0
- package/dist/api/video-request.js +18 -0
- package/dist/api/video.d.ts +22 -2
- package/dist/api/video.js +25 -15
- package/dist/api/vla-helpers.d.ts +1 -2
- package/dist/api/vla-helpers.js +1 -2
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.d.ts +71 -0
- package/dist/api/world-result.js +293 -0
- package/dist/api/world.d.ts +143 -0
- package/dist/api/world.js +71 -0
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.js +15 -28
- package/dist/errors/index.d.ts +35 -31
- package/dist/errors/index.js +66 -56
- package/dist/handlers/assess-model-fit.d.ts +10 -0
- package/dist/handlers/assess-model-fit.js +75 -0
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +1 -1
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/download-asset.js +6 -5
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +0 -12
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/get-system-resources.js +8 -0
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +3 -3
- package/dist/handlers/load-model/download-stats.js +5 -5
- package/dist/handlers/load-model/handler.js +6 -8
- package/dist/handlers/load-model/http-verify.d.ts +20 -0
- package/dist/handlers/load-model/http-verify.js +54 -0
- package/dist/handlers/load-model/http.d.ts +12 -2
- package/dist/handlers/load-model/http.js +119 -230
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +1 -1
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +4 -4
- package/dist/handlers/load-model/resolve-session.d.ts +8 -1
- package/dist/handlers/load-model/resolve-session.js +12 -9
- package/dist/handlers/load-model/resolve.d.ts +4 -4
- package/dist/handlers/load-model/resolve.js +18 -16
- package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
- package/dist/handlers/load-model/safe-fetch.js +143 -0
- package/dist/handlers/load-model/types.d.ts +9 -0
- package/dist/handlers/plugin-dispatch.js +1 -4
- package/dist/handlers/plugin-invoke.js +1 -4
- package/dist/handlers/types.d.ts +0 -2
- package/dist/handlers/unload-model.js +1 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +250 -0
- package/dist/models/registry/models.js +2071 -1806
- package/dist/models/registry/resource-profiles.d.ts +18 -0
- package/dist/models/registry/resource-profiles.js +4934 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +12 -12
- package/dist/plugins/builtin/index.js +12 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +79 -17
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.js +1 -1
- package/dist/plugins/ops/load-model.js +1 -1
- package/dist/plugins/ops/transcribe.js +2 -2
- package/dist/plugins/ops/translate.js +4 -12
- package/dist/plugins/ops/unload-model.js +19 -21
- package/dist/plugins/registry.d.ts +1 -0
- package/dist/plugins/registry.js +12 -0
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/envelope.d.ts +0 -1
- package/dist/profiling/envelope.js +0 -10
- package/dist/profiling/events.d.ts +2 -3
- package/dist/profiling/events.js +2 -9
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -14
- package/dist/profiling/index.js +12 -13
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +35 -2
- package/dist/profiling/operation-wrappers.js +12 -15
- package/dist/profiling/profiler.d.ts +1 -2
- package/dist/profiling/profiler.js +7 -26
- package/dist/profiling/types.d.ts +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +2 -6
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.d.ts +4 -2
- package/dist/rag/rag-workspace-manager.js +268 -15
- package/dist/registry.js +8 -58
- package/dist/resources/collector.d.ts +3 -1
- package/dist/resources/collector.js +91 -9
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/model-fit/assess.d.ts +36 -0
- package/dist/resources/model-fit/assess.js +679 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
- package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
- package/dist/resources/model-fit/calibration/fit.js +152 -0
- package/dist/resources/model-fit/calibration/index.d.ts +37 -0
- package/dist/resources/model-fit/calibration/index.js +75 -0
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
- package/dist/resources/model-fit/estimators/llm.d.ts +46 -0
- package/dist/resources/model-fit/estimators/llm.js +222 -0
- package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
- package/dist/resources/model-fit/estimators/whisper.js +77 -0
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/types.d.ts +120 -0
- package/dist/resources/model-fit/types.js +1 -0
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/native.js +15 -0
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +14 -0
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +5 -7
- package/dist/runtime/model-registry.d.ts +0 -14
- package/dist/runtime/model-registry.js +17 -40
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +13 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +4 -5
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +18 -3
- package/dist/schemas/assess-model-fit.d.ts +311 -0
- package/dist/schemas/assess-model-fit.js +182 -0
- package/dist/schemas/audio-gen.d.ts +437 -7
- package/dist/schemas/audio-gen.js +131 -19
- package/dist/schemas/batch-completion-stream.d.ts +5 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.js +73 -27
- package/dist/schemas/bci.d.ts +3 -3
- package/dist/schemas/bci.js +5 -5
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/classification.js +13 -6
- package/dist/schemas/common.d.ts +695 -335
- package/dist/schemas/common.js +33 -31
- package/dist/schemas/completion-event.d.ts +5 -2
- package/dist/schemas/completion-event.js +2 -1
- package/dist/schemas/completion-stream.d.ts +3 -5
- package/dist/schemas/completion-stream.js +4 -8
- package/dist/schemas/config.d.ts +640 -18
- package/dist/schemas/config.js +50 -8
- package/dist/schemas/download-asset.d.ts +19 -1
- package/dist/schemas/download-asset.js +20 -4
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +3 -3
- package/dist/schemas/error.d.ts +2 -2
- package/dist/schemas/error.js +2 -2
- package/dist/schemas/errors.d.ts +2 -6
- package/dist/schemas/errors.js +30 -40
- package/dist/schemas/heartbeat.d.ts +10 -0
- package/dist/schemas/heartbeat.js +8 -0
- package/dist/schemas/index.d.ts +46 -44
- package/dist/schemas/index.js +45 -43
- package/dist/schemas/llamacpp-config.d.ts +32 -10
- package/dist/schemas/llamacpp-config.js +11 -8
- package/dist/schemas/load-model.d.ts +2902 -1857
- package/dist/schemas/load-model.js +52 -52
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +2354 -0
- package/dist/schemas/model-config-schemas.js +46 -0
- package/dist/schemas/model-info.d.ts +10 -20
- package/dist/schemas/model-info.js +4 -25
- package/dist/schemas/model-ops.d.ts +3 -1
- package/dist/schemas/model-ops.js +7 -2
- package/dist/schemas/model-resource-profile.d.ts +109 -0
- package/dist/schemas/model-resource-profile.js +83 -0
- package/dist/schemas/model-src-utils.d.ts +1 -0
- package/dist/schemas/model-src-utils.js +33 -13
- package/dist/schemas/ocr.js +48 -13
- package/dist/schemas/path-validation.js +8 -2
- package/dist/schemas/plugin.d.ts +23 -10
- package/dist/schemas/plugin.js +11 -0
- package/dist/schemas/profiling.d.ts +138 -43
- package/dist/schemas/profiling.js +1 -22
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +0 -2
- package/dist/schemas/reload-config.js +2 -3
- package/dist/schemas/sdcpp-config.d.ts +344 -4
- package/dist/schemas/sdcpp-config.js +504 -17
- package/dist/schemas/system-resources.d.ts +138 -0
- package/dist/schemas/system-resources.js +3 -1
- package/dist/schemas/text-to-speech.js +183 -76
- package/dist/schemas/transcription-config.js +253 -85
- package/dist/schemas/transcription.d.ts +3 -3
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.js +53 -25
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/selection.d.ts +1 -5
- package/dist/selection.js +2 -6
- package/dist/surface.d.ts +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +1 -1
- package/dist/utils/cache/auto-history.js +1 -1
- package/dist/utils/cache/paths.js +5 -5
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/image-dimensions.d.ts +26 -0
- package/dist/utils/image-dimensions.js +46 -0
- package/dist/utils/index.d.ts +13 -10
- package/dist/utils/index.js +13 -10
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-sanitize.d.ts +7 -0
- package/dist/utils/path-sanitize.js +36 -0
- package/dist/utils/path-security.d.ts +0 -7
- package/dist/utils/path-security.js +1 -36
- package/dist/utils/shard-utils.js +3 -3
- package/dist/utils/tool-integration.js +1 -1
- package/dist/utils/tools/dialect.js +1 -1
- package/dist/utils/tools/index.d.ts +2 -2
- package/dist/utils/tools/index.js +2 -2
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +3 -3
- package/dist/utils/url-security.d.ts +17 -0
- package/dist/utils/url-security.js +75 -0
- package/package.json +31 -26
- package/dist/api/provide.d.ts +0 -23
- package/dist/api/provide.js +0 -32
- package/dist/api/stop-provider.d.ts +0 -20
- package/dist/api/stop-provider.js +0 -30
- package/dist/p2p/cancel-delegated.d.ts +0 -3
- package/dist/p2p/cancel-delegated.js +0 -64
- package/dist/p2p/completion-stream-delegated.d.ts +0 -4
- package/dist/p2p/completion-stream-delegated.js +0 -51
- package/dist/p2p/delegate-client.d.ts +0 -9
- package/dist/p2p/delegate-client.js +0 -219
- package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
- package/dist/p2p/delegate-connect-diagnostics.js +0 -38
- package/dist/p2p/delegate-transport.d.ts +0 -18
- package/dist/p2p/delegate-transport.js +0 -237
- package/dist/p2p/error-handlers.d.ts +0 -4
- package/dist/p2p/error-handlers.js +0 -31
- package/dist/p2p/handle-request.d.ts +0 -2
- package/dist/p2p/handle-request.js +0 -171
- package/dist/p2p/handler-utils.d.ts +0 -19
- package/dist/p2p/handler-utils.js +0 -180
- package/dist/p2p/heartbeat-delegated.d.ts +0 -3
- package/dist/p2p/heartbeat-delegated.js +0 -32
- package/dist/p2p/load-model-delegated.d.ts +0 -6
- package/dist/p2p/load-model-delegated.js +0 -132
- package/dist/p2p/progress-throttle.d.ts +0 -7
- package/dist/p2p/progress-throttle.js +0 -47
- package/dist/p2p/provideHandler/connection.d.ts +0 -2
- package/dist/p2p/provideHandler/connection.js +0 -38
- package/dist/p2p/provideHandler/handler.d.ts +0 -2
- package/dist/p2p/provideHandler/handler.js +0 -66
- package/dist/p2p/provideHandler/proxy.d.ts +0 -2
- package/dist/p2p/provideHandler/proxy.js +0 -24
- package/dist/p2p/provideHandler.d.ts +0 -1
- package/dist/p2p/provideHandler.js +0 -2
- package/dist/p2p/rpc-utils.d.ts +0 -4
- package/dist/p2p/rpc-utils.js +0 -8
- package/dist/p2p/stopProvideHandler.d.ts +0 -2
- package/dist/p2p/stopProvideHandler.js +0 -29
- package/dist/p2p/swarm.d.ts +0 -9
- package/dist/p2p/swarm.js +0 -103
- package/dist/p2p/transport-selector.d.ts +0 -2
- package/dist/p2p/transport-selector.js +0 -18
- package/dist/p2p/unload-model-delegated.d.ts +0 -2
- package/dist/p2p/unload-model-delegated.js +0 -31
- package/dist/profiling/delegation-profiler.d.ts +0 -31
- package/dist/profiling/delegation-profiler.js +0 -155
- package/dist/schemas/delegate.d.ts +0 -29
- package/dist/schemas/delegate.js +0 -40
- package/dist/schemas/provide.d.ts +0 -50
- package/dist/schemas/provide.js +0 -32
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from '
|
|
2
|
-
import { extractSystemPrompt, getCurrentCacheInfo } from '
|
|
1
|
+
import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from './cache-logger.js';
|
|
2
|
+
import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
|
|
3
|
+
import { isAddonPreMutationRefusal, isAddonContextOverflowError } from './context-overflow.js';
|
|
3
4
|
import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
|
|
4
|
-
import { decideCachedHistorySlice, shouldCommitCachedTurn } from '
|
|
5
|
-
import { createKvCacheSession, generateConfigHash } from '
|
|
5
|
+
import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
|
|
6
|
+
import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
|
|
6
7
|
import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
7
8
|
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
8
9
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
@@ -11,7 +12,7 @@ import { getEngineLogger } from '../../../../logging/index.js';
|
|
|
11
12
|
import { AttachmentNotFoundError } from '../../../../errors/index.js';
|
|
12
13
|
import { nowMs } from '../../../../profiling/index.js';
|
|
13
14
|
import { buildStreamResult } from '../../../../profiling/model-execution.js';
|
|
14
|
-
import { normalizeCompletionStats, withEmittedTokens } from '
|
|
15
|
+
import { normalizeCompletionStats, withEmittedTokens } from './completion-stats.js';
|
|
15
16
|
import fs from 'bare-fs';
|
|
16
17
|
const logger = getEngineLogger();
|
|
17
18
|
function transformMessage(message) {
|
|
@@ -119,7 +120,7 @@ function rendersToolBlock(messages, toolBlock) {
|
|
|
119
120
|
* - The tool block travels only with the turn that writes it into the
|
|
120
121
|
* cache; see `skipToolBlock` below.
|
|
121
122
|
*/
|
|
122
|
-
function prepareMessagesForCache(session, turn, cacheExists, history, tools
|
|
123
|
+
function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
|
|
123
124
|
const toolBlock = tools?.length ? transformMessages(tools) : [];
|
|
124
125
|
if (!(cacheExists && history.length > 0)) {
|
|
125
126
|
const historyWithoutSystem = history.filter((msg) => msg.role !== 'system');
|
|
@@ -146,9 +147,8 @@ function prepareMessagesForCache(session, turn, cacheExists, history, tools, too
|
|
|
146
147
|
// conversation. Skip it only when the prefix is known to hold a rendered
|
|
147
148
|
// one: `toolBlockCached` records that a previous turn actually got it into
|
|
148
149
|
// the cache, which a committed message count does not prove. A stale
|
|
149
|
-
// boundary means we are resending the whole conversation anyway
|
|
150
|
-
|
|
151
|
-
const skipToolBlock = turn.toolBlockCached && !clearStaleCount && !toolBlockEvictable;
|
|
150
|
+
// boundary means we are resending the whole conversation anyway.
|
|
151
|
+
const skipToolBlock = turn.toolBlockCached && !clearStaleCount;
|
|
152
152
|
const blockToSend = skipToolBlock ? [] : toolBlock;
|
|
153
153
|
return {
|
|
154
154
|
messages: withToolBlock(transformMessages(messages), blockToSend),
|
|
@@ -208,12 +208,6 @@ export async function* completion(params, opts) {
|
|
|
208
208
|
const modelConfig = getModelConfig(modelId);
|
|
209
209
|
const toolsEnabled = modelConfig.tools === true;
|
|
210
210
|
const toolsActive = !!tools?.length && toolsEnabled;
|
|
211
|
-
// Sliding is opt-in (`n_discarded` defaults to 0). Once on, the addon's
|
|
212
|
-
// discard window opens at the end of the primed prefix — which is where the
|
|
213
|
-
// tool block sits, since the prime is the system prompt alone — and nothing
|
|
214
|
-
// protects it. So while sliding is possible the block cannot be assumed to
|
|
215
|
-
// survive, and it has to travel with every turn.
|
|
216
|
-
const toolBlockEvictable = (modelConfig.n_discarded ?? 0) > 0;
|
|
217
211
|
const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
|
|
218
212
|
// `responseFormat` is forwarded to the addon as a per-request
|
|
219
213
|
// `generationParams.json_schema`, which the addon converts to GBNF and
|
|
@@ -273,12 +267,11 @@ export async function* completion(params, opts) {
|
|
|
273
267
|
logMessagesToAddon(transformedHistory, 'NO_CACHE');
|
|
274
268
|
return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect, setActiveResponse);
|
|
275
269
|
}
|
|
276
|
-
// ---- KV-cache path. The session owns
|
|
277
|
-
//
|
|
278
|
-
//
|
|
279
|
-
// the happy path
|
|
280
|
-
//
|
|
281
|
-
// unwind through the same `scope.defer` hook. ----
|
|
270
|
+
// ---- KV-cache path. The session owns every bookkeeping layer; the
|
|
271
|
+
// handler registers one deferred unwind (`rollback`, or the non-destructive
|
|
272
|
+
// `releaseTurn` on a recognised pre-mutation refusal) that `commitTurn`
|
|
273
|
+
// short-circuits on the happy path. Cancellations / zero-token replies /
|
|
274
|
+
// rename failures all still unwind destructively through the same hook. ----
|
|
282
275
|
const session = createKvCacheSession(modelId, { logger: requestLogger });
|
|
283
276
|
const systemPromptFromHistory = extractSystemPrompt(history);
|
|
284
277
|
// The tool block is baked into the cache on the turn that first sends it and
|
|
@@ -319,16 +312,34 @@ export async function* completion(params, opts) {
|
|
|
319
312
|
// flips the turn's internal `committed` flag so this becomes a no-op
|
|
320
313
|
// on the happy path. Scope unwinding is LIFO — registered after the
|
|
321
314
|
// `removeEventListener` defer above so rollback runs before the
|
|
322
|
-
// listener detach.
|
|
323
|
-
|
|
315
|
+
// listener detach. A thrown overflow, pre-mutation refusal, or pre-addon
|
|
316
|
+
// attachment failure never persists the turn; the committed cache stays valid.
|
|
317
|
+
let preserveCacheOnUnwind = false;
|
|
318
|
+
scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
|
|
324
319
|
// `cacheExists` is implied by `beginTurn` — the session either found
|
|
325
320
|
// an existing cache or just primed one. Pass `true` to the message
|
|
326
321
|
// selector so the slicing branches engage.
|
|
327
|
-
|
|
328
|
-
|
|
322
|
+
let payload;
|
|
323
|
+
try {
|
|
324
|
+
payload = prepareMessagesForCache(session, turn,
|
|
325
|
+
/* cacheExists */ true, history, toolsActive ? tools : undefined);
|
|
326
|
+
}
|
|
327
|
+
catch (error) {
|
|
328
|
+
// A missing attachment is caller input rejected before the addon runs,
|
|
329
|
+
// so the committed cache is untouched and must survive.
|
|
330
|
+
preserveCacheOnUnwind = error instanceof AttachmentNotFoundError;
|
|
331
|
+
throw error;
|
|
332
|
+
}
|
|
329
333
|
const messagesToSend = payload.messages;
|
|
330
334
|
logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
|
|
331
|
-
|
|
335
|
+
let result;
|
|
336
|
+
try {
|
|
337
|
+
result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse);
|
|
338
|
+
}
|
|
339
|
+
catch (error) {
|
|
340
|
+
preserveCacheOnUnwind = isAddonContextOverflowError(error) || isAddonPreMutationRefusal(error);
|
|
341
|
+
throw error;
|
|
342
|
+
}
|
|
332
343
|
const shouldCommitTurn = shouldCommitCachedTurn({
|
|
333
344
|
aborted: signal.aborted,
|
|
334
345
|
producedTokens: result.producedTokens,
|
|
@@ -6,23 +6,24 @@
|
|
|
6
6
|
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
7
7
|
* `qvac_errors::StatusError::codeString()` and formats as
|
|
8
8
|
* `"[ <addonId> :: ContextOverflow ]"`. The message carries the
|
|
9
|
-
* C++-formatted detail (which may include the prompt/ctx sizes).
|
|
9
|
+
* C++-formatted detail (which may include the prompt/ctx sizes). That
|
|
10
|
+
* codeString survives only the synchronous throw path — asynchronous run
|
|
11
|
+
* errors are transported as `exception.what()` alone, with no code.
|
|
10
12
|
*
|
|
11
13
|
* These helpers let the plugin handler convert that addon error into a
|
|
12
14
|
* typed `ContextOverflowError` and let unit tests assert the detection
|
|
13
15
|
* + extraction logic without a real model load.
|
|
14
16
|
*/
|
|
15
17
|
export declare function isAddonContextOverflowError(err: unknown): boolean;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
* `MtmdLlmContext.cpp` format both numbers into the message; the
|
|
20
|
-
* `LlamaModel::processPromptImpl` fallback path emits a bare
|
|
21
|
-
* `"<func>: context overflow\n"` with neither number. Returning
|
|
22
|
-
* `undefined` for either field is fine — `ContextOverflowError` holds
|
|
23
|
-
* them as optional and the message factory degrades gracefully.
|
|
24
|
-
*/
|
|
25
|
-
export declare function parseContextOverflowMessage(message: string): {
|
|
18
|
+
export declare function isAddonPreMutationRefusal(err: unknown): boolean;
|
|
19
|
+
export type ContextOverflowSizes = {
|
|
20
|
+
/** The prompt alone, in tokens; unset when the guard reports KV cells. */
|
|
26
21
|
promptTokens?: number;
|
|
22
|
+
/** Cached conversation on a warm-cache guard, in `ctxSize` units. */
|
|
23
|
+
cachedTokens?: number;
|
|
24
|
+
/** Total the failing guard reported, in `ctxSize` units; the guards trigger on `>=`, so it can equal the window. */
|
|
25
|
+
requiredTokens?: number;
|
|
26
|
+
/** Effective per-request ceiling: `ctx_size` split across slots at `parallel > 1`. */
|
|
27
27
|
ctxSize?: number;
|
|
28
28
|
};
|
|
29
|
+
export declare function parseContextOverflowMessage(message: string): ContextOverflowSizes;
|
|
@@ -6,60 +6,152 @@
|
|
|
6
6
|
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
7
7
|
* `qvac_errors::StatusError::codeString()` and formats as
|
|
8
8
|
* `"[ <addonId> :: ContextOverflow ]"`. The message carries the
|
|
9
|
-
* C++-formatted detail (which may include the prompt/ctx sizes).
|
|
9
|
+
* C++-formatted detail (which may include the prompt/ctx sizes). That
|
|
10
|
+
* codeString survives only the synchronous throw path — asynchronous run
|
|
11
|
+
* errors are transported as `exception.what()` alone, with no code.
|
|
10
12
|
*
|
|
11
13
|
* These helpers let the plugin handler convert that addon error into a
|
|
12
14
|
* typed `ContextOverflowError` and let unit tests assert the detection
|
|
13
15
|
* + extraction logic without a real model load.
|
|
14
16
|
*/
|
|
17
|
+
// The guards' emitted starts; the numeric tails vary across addon
|
|
18
|
+
// generations, so the forms are start-anchored and single-line.
|
|
19
|
+
const CONTEXT_OVERFLOW_FORMS = [
|
|
20
|
+
/^\[TextLlm\] context overflow at (?:batch )?prefill step[^\r\n]*$/,
|
|
21
|
+
/^\[MtmdLlm\] context overflow at (?:batch )?prefill step[^\r\n]*$/,
|
|
22
|
+
// Unreachable at the pinned addon generation (generateResponse never
|
|
23
|
+
// reports !ok); kept in case a patch revives the path.
|
|
24
|
+
/^processPromptImpl: context overflow$/,
|
|
25
|
+
// The batch scheduler's capacity refusals are the same out-of-context
|
|
26
|
+
// condition, reported under a generic status.
|
|
27
|
+
/^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
28
|
+
/^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
29
|
+
/^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
30
|
+
/^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/
|
|
31
|
+
];
|
|
32
|
+
// Both detectors classify raw addon errors only — an already-typed QvacError
|
|
33
|
+
// (numeric `code`) is deliberately rejected by the present-code gate.
|
|
15
34
|
export function isAddonContextOverflowError(err) {
|
|
16
35
|
if (typeof err !== 'object' || err === null)
|
|
17
36
|
return false;
|
|
37
|
+
// A present status code is authoritative: exact ContextOverflow accepts,
|
|
38
|
+
// anything else (a non-string included) rejects with no message fallback.
|
|
18
39
|
const code = err.code;
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
// `ContextOverflowRecovered` or `PostContextOverflow` after an
|
|
22
|
-
// addon-side rename.
|
|
23
|
-
if (typeof code === 'string' && /::\s*ContextOverflow\s*\]/.test(code)) {
|
|
24
|
-
return true;
|
|
40
|
+
if (code !== undefined) {
|
|
41
|
+
return typeof code === 'string' && /^\[\s*[\w.-]+\s*::\s*ContextOverflow\s*\]$/.test(code);
|
|
25
42
|
}
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
// two known C++-emitted formats only — broader substring would catch
|
|
29
|
-
// wrapper / cause-chain text that mentions overflow without being one.
|
|
43
|
+
// The production path: the async transport strips status metadata from
|
|
44
|
+
// every run error, so the message alone identifies the overflow guards.
|
|
30
45
|
const message = err.message;
|
|
31
|
-
|
|
32
|
-
|
|
46
|
+
if (typeof message !== 'string')
|
|
47
|
+
return false;
|
|
48
|
+
const trimmed = message.trim();
|
|
49
|
+
return CONTEXT_OVERFLOW_FORMS.some((form) => form.test(trimmed));
|
|
33
50
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
*
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
// Refusals thrown before any decode or disk save; the async transport delivers
|
|
52
|
+
// exception.what() alone, so each form is matched complete and end-anchored.
|
|
53
|
+
const PRE_MUTATION_REFUSAL_FORMS = [
|
|
54
|
+
/^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
55
|
+
/^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
56
|
+
/^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
57
|
+
/^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
58
|
+
/^ContinuousBatchScheduler::submit: failed to add to batch \(MultiRequestBatcher::AddStatus=[1-4]\)$/,
|
|
59
|
+
/^invalid generationParams\.json_schema: [^\r\n]*$/,
|
|
60
|
+
/^failed to initialise sampler with per-request generationParams \(invalid grammar or json_schema\?\)$/,
|
|
61
|
+
/^\[MtmdLlm\] Media buffer is empty$/,
|
|
62
|
+
/^\[MtmdLlm\] Filename is empty$/,
|
|
63
|
+
/^\[MtmdLlm\] Failed to load media from file: [^\r\n]*$/,
|
|
64
|
+
/^\[MtmdLlm\] preparePrefill: prompt must end with text after the last media item$/
|
|
65
|
+
];
|
|
66
|
+
export function isAddonPreMutationRefusal(err) {
|
|
67
|
+
if (typeof err !== 'object' || err === null)
|
|
68
|
+
return false;
|
|
69
|
+
// A status code only exists on the synchronous throw path; when present it
|
|
70
|
+
// must be InvalidArgument, and its absence (the async transport) is fine.
|
|
71
|
+
const code = err.code;
|
|
72
|
+
if (code !== undefined &&
|
|
73
|
+
(typeof code !== 'string' || !/^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code))) {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
const message = err.message;
|
|
77
|
+
if (typeof message !== 'string')
|
|
78
|
+
return false;
|
|
79
|
+
const trimmed = message.trim();
|
|
80
|
+
return PRE_MUTATION_REFUSAL_FORMS.some((form) => form.test(trimmed));
|
|
81
|
+
}
|
|
82
|
+
// One entry per number-formatting guard, most specific first; keep in step
|
|
83
|
+
// with TextLlmContext.cpp / MtmdLlmContext.cpp or fields silently vanish.
|
|
84
|
+
const MESSAGE_PATTERNS = [
|
|
85
|
+
{
|
|
86
|
+
// "cached tokens C plus prompt tokens N exceed the max context tokens M"
|
|
87
|
+
pattern: /cached tokens (\d+) plus prompt tokens (\d+) exceeds? the max context tokens (\d+)/i,
|
|
88
|
+
map: ([cached, prompt, ctx]) => ({
|
|
89
|
+
promptTokens: prompt,
|
|
90
|
+
cachedTokens: cached,
|
|
91
|
+
requiredTokens: cached + prompt,
|
|
92
|
+
ctxSize: ctx
|
|
93
|
+
})
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
// "cached P positions / C KV cells plus ... of prompt exceed the max
|
|
97
|
+
// context tokens M" — KV cells, not tokens, so promptTokens stays unset.
|
|
98
|
+
pattern: /cached (\d+) positions \/ (\d+) KV cells plus (\d+) positions \/ (\d+) KV cells of prompt exceeds? the max context tokens (\d+)/i,
|
|
99
|
+
map: ([cPos, cCells, pPos, pCells, ctx]) => ({
|
|
100
|
+
cachedTokens: Math.max(cPos, cCells),
|
|
101
|
+
requiredTokens: Math.max(cPos + pPos, cCells + pCells),
|
|
102
|
+
ctxSize: ctx
|
|
103
|
+
})
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
// "prompt tokens N, max context tokens M"
|
|
107
|
+
pattern: /prompt tokens (\d+)[, \t]+max context tokens (\d+)/i,
|
|
108
|
+
map: ([prompt, ctx]) => ({ promptTokens: prompt, requiredTokens: prompt, ctxSize: ctx })
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
// "prompt spans P positions / N KV cells, max context tokens M" —
|
|
112
|
+
// not tokens, so promptTokens stays unset.
|
|
113
|
+
pattern: /prompt spans (\d+) positions \/ (\d+) KV cells,[ \t]*max context tokens (\d+)/i,
|
|
114
|
+
map: ([pos, cells, ctx]) => ({ requiredTokens: Math.max(pos, cells), ctxSize: ctx })
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
// "(N tokens, P positions, max M)" — the "tokens" figure is
|
|
118
|
+
// mtmd_helper_get_n_tokens, i.e. KV cells, so promptTokens stays unset.
|
|
119
|
+
pattern: /\((\d+)[ \t]+tokens,[ \t]*(\d+)[ \t]+positions,[ \t]*max[ \t]+(\d+)\)/i,
|
|
120
|
+
map: ([cells, pos, ctx]) => ({
|
|
121
|
+
requiredTokens: Math.max(cells, pos),
|
|
122
|
+
ctxSize: ctx
|
|
123
|
+
})
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
// "(N tokens, max M)", the retired short form. Both of its emitters
|
|
127
|
+
// format a cached total, not the prompt alone — promptTokens stays unset.
|
|
128
|
+
pattern: /\((\d+)[ \t]+tokens,[ \t]*max[ \t]+(\d+)\)/i,
|
|
129
|
+
map: ([total, ctx]) => ({ requiredTokens: total, ctxSize: ctx })
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
// Scheduler cap refusals: the total is cache-plus-prompt in cells or
|
|
133
|
+
// tokens, and the cap is the effective per-request ceiling.
|
|
134
|
+
pattern: /prompt of (\d+) (?:KV cells|tokens) (?:exceeds|leaves no room under) per-sequence cap (\d+)/,
|
|
135
|
+
map: ([total, cap]) => ({ requiredTokens: total, ctxSize: cap })
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
// "n_predict P + prompt N KV cells exceeds per-sequence cap M" — the
|
|
139
|
+
// reservation plus the prompt is the space the request needs.
|
|
140
|
+
pattern: /n_predict (\d+) \+ prompt (\d+) KV cells exceeds per-sequence cap (\d+)/,
|
|
141
|
+
map: ([predict, cells, cap]) => ({ requiredTokens: predict + cells, ctxSize: cap })
|
|
54
142
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
143
|
+
];
|
|
144
|
+
// Best-effort extraction of the overflow sizes; the bare processPromptImpl
|
|
145
|
+
// path carries no numbers, and undefined fields degrade gracefully.
|
|
146
|
+
export function parseContextOverflowMessage(message) {
|
|
147
|
+
for (const { pattern, map } of MESSAGE_PATTERNS) {
|
|
148
|
+
const match = message.match(pattern);
|
|
149
|
+
if (!match)
|
|
150
|
+
continue;
|
|
151
|
+
const numbers = match.slice(1).map(Number);
|
|
152
|
+
if (!numbers.every(Number.isFinite))
|
|
153
|
+
continue;
|
|
154
|
+
return map(numbers);
|
|
63
155
|
}
|
|
64
156
|
return {};
|
|
65
157
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { generateConfigHash } from '
|
|
1
|
+
import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
|
|
2
2
|
import { type CacheMessage } from '../../../../utils/index.js';
|
|
3
3
|
import type { Logger } from '../../../../logging/types.js';
|
|
4
4
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
@@ -106,6 +106,12 @@ export interface KvCacheSession {
|
|
|
106
106
|
* itself via `commitTurn`).
|
|
107
107
|
*/
|
|
108
108
|
rollback(turn: TurnHandle): Promise<void>;
|
|
109
|
+
/**
|
|
110
|
+
* Non-destructive counterpart of `rollback`: releases locks and refs but
|
|
111
|
+
* keeps the committed disk cache and its recorded prefix valid for a retry.
|
|
112
|
+
* A cache freshly primed by this same turn rolls back instead.
|
|
113
|
+
*/
|
|
114
|
+
releaseTurn(turn: TurnHandle): Promise<void>;
|
|
109
115
|
/**
|
|
110
116
|
* Forget the in-memory saved-message count for the turn's path
|
|
111
117
|
* without unlinking the file or clearing the init flag. Used when
|
|
@@ -126,9 +132,10 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
126
132
|
}): KvCacheSession;
|
|
127
133
|
/**
|
|
128
134
|
* Atomically delete every layer of KV-cache state for a
|
|
129
|
-
* `(kvCacheKey, modelId)` pair,
|
|
130
|
-
*
|
|
131
|
-
* turn-scoped
|
|
135
|
+
* `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
|
|
136
|
+
* caches no turn is holding. Single entry point — the only mutation
|
|
137
|
+
* point for cross-model state outside of turn-scoped
|
|
138
|
+
* `commitTurn`/`rollback`.
|
|
132
139
|
*
|
|
133
140
|
* Why this isn't a method on `KvCacheSession`: deletes are
|
|
134
141
|
* cross-model (`all: true` has no model; the keyed form has
|
|
@@ -155,12 +162,16 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
155
162
|
* its file splits state: the file stays on disk while its saved-count and
|
|
156
163
|
* init flag are cleared, so the next turn sees the file, skips priming, and
|
|
157
164
|
* reports `savedCount=0`. Callers must not delete a key that is in active use.
|
|
165
|
+
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
166
|
+
* and clears each one under the cache-state lock.
|
|
158
167
|
*/
|
|
159
168
|
export declare function deleteKvCacheState(target: {
|
|
160
169
|
kvCacheKey: string;
|
|
161
170
|
modelId?: string;
|
|
162
171
|
} | {
|
|
163
172
|
all: true;
|
|
173
|
+
} | {
|
|
174
|
+
auto: true;
|
|
164
175
|
}): Promise<void>;
|
|
165
176
|
/**
|
|
166
177
|
* Test-only access to the module-scoped state. Production code reaches
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
|
-
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '
|
|
4
|
-
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '
|
|
3
|
+
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../ops/kv-cache-utils.js';
|
|
4
|
+
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../ops/kv-cache-retention.js';
|
|
5
5
|
import { isMobile } from '../../../../runtime/state.js';
|
|
6
6
|
import { getKVCacheDir } from '../../../../utils/index.js';
|
|
7
|
-
import { logCacheSaveError, logCacheStatus } from '
|
|
7
|
+
import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
|
|
8
8
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
9
9
|
// Used by cross-model paths that have no `RequestContext` (e.g.
|
|
10
10
|
// `deleteKvCacheState`). Per-session call sites receive a logger from
|
|
@@ -128,6 +128,25 @@ function isCacheKeyActive(cacheKey) {
|
|
|
128
128
|
function getAutoCacheMaxBytes() {
|
|
129
129
|
return isMobile() ? MOBILE_AUTO_CACHE_MAX_BYTES : DESKTOP_AUTO_CACHE_MAX_BYTES;
|
|
130
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* Deletes planned auto-cache keys and reports how many went.
|
|
133
|
+
*
|
|
134
|
+
* Planning and eviction are separate steps, so a turn can take a key in
|
|
135
|
+
* between: each one is re-checked against the active paths under the
|
|
136
|
+
* cache-state lock rather than trusted from the plan.
|
|
137
|
+
*/
|
|
138
|
+
async function evictPlannedAutoCaches(cacheKeys) {
|
|
139
|
+
let evicted = 0;
|
|
140
|
+
await withCacheStateLock(async () => {
|
|
141
|
+
for (const cacheKey of cacheKeys) {
|
|
142
|
+
if (isCacheKeyActive(cacheKey))
|
|
143
|
+
continue;
|
|
144
|
+
await deleteKvCacheState({ kvCacheKey: cacheKey });
|
|
145
|
+
evicted++;
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
return evicted;
|
|
149
|
+
}
|
|
131
150
|
async function withCacheStateLock(operation) {
|
|
132
151
|
const previous = cacheStateLockTail;
|
|
133
152
|
let releaseLock = () => { };
|
|
@@ -156,20 +175,12 @@ async function maybeSweepAutoCaches(logger, overrides) {
|
|
|
156
175
|
const sweep = async () => {
|
|
157
176
|
try {
|
|
158
177
|
const cacheKeys = await planAutoCacheEvictions({
|
|
159
|
-
activeCachePaths:
|
|
178
|
+
activeCachePaths: snapshotActivePaths(),
|
|
160
179
|
maxBytes: overrides?.maxBytes ?? getAutoCacheMaxBytes(),
|
|
161
180
|
maxIdleMs: overrides?.maxIdleMs ?? AUTO_CACHE_MAX_IDLE_MS,
|
|
162
181
|
nowMs
|
|
163
182
|
});
|
|
164
|
-
|
|
165
|
-
await withCacheStateLock(async () => {
|
|
166
|
-
for (const cacheKey of cacheKeys) {
|
|
167
|
-
if (isCacheKeyActive(cacheKey))
|
|
168
|
-
continue;
|
|
169
|
-
await deleteKvCacheState({ kvCacheKey: cacheKey });
|
|
170
|
-
evictionCount++;
|
|
171
|
-
}
|
|
172
|
-
});
|
|
183
|
+
const evictionCount = await evictPlannedAutoCaches(cacheKeys);
|
|
173
184
|
if (evictionCount > 0) {
|
|
174
185
|
logger.debug(`[kv-cache] Evicted ${evictionCount} inactive auto-cache entries`);
|
|
175
186
|
}
|
|
@@ -192,6 +203,30 @@ function scheduleAutoCacheSweep(logger) {
|
|
|
192
203
|
logger.warn(`[kv-cache] Failed to schedule auto-cache retention sweep: ${error instanceof Error ? error.message : String(error)}`);
|
|
193
204
|
});
|
|
194
205
|
}
|
|
206
|
+
/**
|
|
207
|
+
* Drops every auto cache no turn is holding, whatever its age or size.
|
|
208
|
+
*
|
|
209
|
+
* The on-demand counterpart to `maybeSweepAutoCaches`, which applies the
|
|
210
|
+
* standing quota and TTL instead. Reusing the planner keeps the "is this an
|
|
211
|
+
* auto cache" decision in one place: only `.auto-cache-<key>` markers are
|
|
212
|
+
* considered, so caller-owned named caches stay out of scope.
|
|
213
|
+
*/
|
|
214
|
+
async function deleteInactiveAutoCaches() {
|
|
215
|
+
const cacheKeys = await planAutoCacheEvictions({
|
|
216
|
+
activeCachePaths: snapshotActivePaths(),
|
|
217
|
+
// The quota pass stops at `retainedBytes <= maxBytes`, and a total reaches
|
|
218
|
+
// 0 while zero-byte caches remain (an empty `.bin` from a crashed prime).
|
|
219
|
+
// A negative quota is unreachable, so every inactive entry is selected.
|
|
220
|
+
maxBytes: -1,
|
|
221
|
+
// No age threshold: the quota pass above already selects everything.
|
|
222
|
+
maxIdleMs: 0,
|
|
223
|
+
nowMs: Date.now()
|
|
224
|
+
});
|
|
225
|
+
const evicted = await evictPlannedAutoCaches(cacheKeys);
|
|
226
|
+
if (evicted > 0) {
|
|
227
|
+
moduleLogger.debug(`[kv-cache] Reclaimed ${evicted} inactive auto-cache entries`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
195
230
|
// ----- factory -----
|
|
196
231
|
/**
|
|
197
232
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
@@ -218,7 +253,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
218
253
|
...(signal !== undefined && { signal }),
|
|
219
254
|
releaseWriteLock,
|
|
220
255
|
committed: false,
|
|
221
|
-
rolledBack: false
|
|
256
|
+
rolledBack: false,
|
|
257
|
+
freshlyPrimed: false
|
|
222
258
|
});
|
|
223
259
|
markCachePathActive(cachePath);
|
|
224
260
|
return handle;
|
|
@@ -277,6 +313,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
277
313
|
await input.primeIfMissing(cachePath);
|
|
278
314
|
await verifyPrimedFile(cachePath, logger);
|
|
279
315
|
initializedCaches.add(cachePath);
|
|
316
|
+
turnState.get(handle).freshlyPrimed = true;
|
|
280
317
|
}
|
|
281
318
|
return handle;
|
|
282
319
|
}
|
|
@@ -362,6 +399,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
362
399
|
await input.primeIfMissing(cachePath);
|
|
363
400
|
await verifyPrimedFile(cachePath, logger);
|
|
364
401
|
initializedCaches.add(cachePath);
|
|
402
|
+
turnState.get(handle).freshlyPrimed = true;
|
|
365
403
|
}
|
|
366
404
|
return handle;
|
|
367
405
|
}
|
|
@@ -485,6 +523,24 @@ export function createKvCacheSession(modelId, options) {
|
|
|
485
523
|
return;
|
|
486
524
|
await runRollback(state);
|
|
487
525
|
}
|
|
526
|
+
async function releaseTurn(turn) {
|
|
527
|
+
const state = turnState.get(turn);
|
|
528
|
+
if (!state)
|
|
529
|
+
return;
|
|
530
|
+
if (state.committed || state.rolledBack)
|
|
531
|
+
return;
|
|
532
|
+
// A cache this same turn primed has no committed state to keep — a failed
|
|
533
|
+
// first turn must not leave its own prime behind.
|
|
534
|
+
if (state.freshlyPrimed) {
|
|
535
|
+
await runRollback(state);
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
releaseCachePath(state.cachePath);
|
|
539
|
+
state.rolledBack = true;
|
|
540
|
+
state.releaseWriteLock();
|
|
541
|
+
if (state.autoCacheKey !== undefined)
|
|
542
|
+
scheduleAutoCacheSweep(logger);
|
|
543
|
+
}
|
|
488
544
|
async function runRollback(state) {
|
|
489
545
|
// Order matters only weakly: unlink first so a partial disk-state
|
|
490
546
|
// can't be re-loaded by a sibling turn between the file delete and
|
|
@@ -520,15 +576,17 @@ export function createKvCacheSession(modelId, options) {
|
|
|
520
576
|
beginTurn,
|
|
521
577
|
commitTurn,
|
|
522
578
|
rollback,
|
|
579
|
+
releaseTurn,
|
|
523
580
|
dropStaleSavedCount
|
|
524
581
|
};
|
|
525
582
|
}
|
|
526
583
|
// ----- module-level administrative API -----
|
|
527
584
|
/**
|
|
528
585
|
* Atomically delete every layer of KV-cache state for a
|
|
529
|
-
* `(kvCacheKey, modelId)` pair,
|
|
530
|
-
*
|
|
531
|
-
* turn-scoped
|
|
586
|
+
* `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
|
|
587
|
+
* caches no turn is holding. Single entry point — the only mutation
|
|
588
|
+
* point for cross-model state outside of turn-scoped
|
|
589
|
+
* `commitTurn`/`rollback`.
|
|
532
590
|
*
|
|
533
591
|
* Why this isn't a method on `KvCacheSession`: deletes are
|
|
534
592
|
* cross-model (`all: true` has no model; the keyed form has
|
|
@@ -555,8 +613,12 @@ export function createKvCacheSession(modelId, options) {
|
|
|
555
613
|
* its file splits state: the file stays on disk while its saved-count and
|
|
556
614
|
* init flag are cleared, so the next turn sees the file, skips priming, and
|
|
557
615
|
* reports `savedCount=0`. Callers must not delete a key that is in active use.
|
|
616
|
+
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
617
|
+
* and clears each one under the cache-state lock.
|
|
558
618
|
*/
|
|
559
619
|
export async function deleteKvCacheState(target) {
|
|
620
|
+
if ('auto' in target)
|
|
621
|
+
return deleteInactiveAutoCaches();
|
|
560
622
|
if ('all' in target) {
|
|
561
623
|
const removed = await deleteCacheUtil({ all: true });
|
|
562
624
|
cachedPrefixes.clear();
|