@qvac/inference 0.18.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -2
- package/README.md +1 -5
- package/dist/api/assess-model-fit.d.ts +16 -0
- package/dist/api/assess-model-fit.js +27 -0
- package/dist/api/audio-gen.d.ts +4 -4
- package/dist/api/audio-gen.js +17 -6
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/download-asset.d.ts +1 -1
- package/dist/api/download-asset.js +2 -2
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-loaded-model-info.d.ts +6 -10
- package/dist/api/get-loaded-model-info.js +6 -10
- package/dist/api/heartbeat.d.ts +5 -22
- package/dist/api/heartbeat.js +6 -23
- package/dist/api/index.d.ts +36 -35
- package/dist/api/index.js +35 -35
- package/dist/api/load-model.d.ts +5 -1
- package/dist/api/load-model.js +3 -5
- package/dist/api/logging-stream-registry.js +1 -1
- package/dist/api/resume.d.ts +0 -3
- package/dist/api/resume.js +0 -3
- package/dist/api/subscribe-logs.js +1 -1
- package/dist/api/suspend.d.ts +0 -4
- package/dist/api/suspend.js +0 -4
- package/dist/api/translate.d.ts +3 -1
- package/dist/api/translate.js +16 -8
- package/dist/api/unload-model.js +3 -5
- package/dist/api/video-request.d.ts +207 -0
- package/dist/api/video-request.js +18 -0
- package/dist/api/video.d.ts +22 -2
- package/dist/api/video.js +25 -15
- package/dist/api/vla-helpers.d.ts +1 -2
- package/dist/api/vla-helpers.js +1 -2
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.d.ts +71 -0
- package/dist/api/world-result.js +293 -0
- package/dist/api/world.d.ts +143 -0
- package/dist/api/world.js +71 -0
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.js +15 -28
- package/dist/errors/index.d.ts +35 -31
- package/dist/errors/index.js +66 -56
- package/dist/handlers/assess-model-fit.d.ts +10 -0
- package/dist/handlers/assess-model-fit.js +75 -0
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +1 -1
- package/dist/handlers/download-asset.js +6 -5
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +0 -12
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/get-system-resources.js +8 -0
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +3 -3
- package/dist/handlers/load-model/download-stats.js +5 -5
- package/dist/handlers/load-model/handler.js +6 -8
- package/dist/handlers/load-model/http-verify.d.ts +20 -0
- package/dist/handlers/load-model/http-verify.js +54 -0
- package/dist/handlers/load-model/http.d.ts +12 -2
- package/dist/handlers/load-model/http.js +119 -230
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +1 -1
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +4 -4
- package/dist/handlers/load-model/resolve-session.d.ts +8 -1
- package/dist/handlers/load-model/resolve-session.js +12 -9
- package/dist/handlers/load-model/resolve.d.ts +4 -4
- package/dist/handlers/load-model/resolve.js +18 -16
- package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
- package/dist/handlers/load-model/safe-fetch.js +143 -0
- package/dist/handlers/load-model/types.d.ts +9 -0
- package/dist/handlers/plugin-dispatch.js +1 -4
- package/dist/handlers/plugin-invoke.js +1 -4
- package/dist/handlers/types.d.ts +0 -2
- package/dist/handlers/unload-model.js +1 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +250 -0
- package/dist/models/registry/models.js +2071 -1806
- package/dist/models/registry/resource-profiles.d.ts +18 -0
- package/dist/models/registry/resource-profiles.js +4934 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +12 -12
- package/dist/plugins/builtin/index.js +12 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +7 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +26 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +19 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.js +1 -1
- package/dist/plugins/ops/load-model.js +1 -1
- package/dist/plugins/ops/transcribe.js +2 -2
- package/dist/plugins/ops/translate.js +4 -12
- package/dist/plugins/ops/unload-model.js +19 -21
- package/dist/plugins/registry.d.ts +1 -0
- package/dist/plugins/registry.js +12 -0
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/envelope.d.ts +0 -1
- package/dist/profiling/envelope.js +0 -10
- package/dist/profiling/events.d.ts +2 -3
- package/dist/profiling/events.js +2 -9
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -14
- package/dist/profiling/index.js +12 -13
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +35 -2
- package/dist/profiling/operation-wrappers.js +12 -15
- package/dist/profiling/profiler.d.ts +1 -2
- package/dist/profiling/profiler.js +7 -26
- package/dist/profiling/types.d.ts +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +2 -6
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.d.ts +4 -2
- package/dist/rag/rag-workspace-manager.js +268 -15
- package/dist/registry.js +8 -58
- package/dist/resources/collector.d.ts +3 -1
- package/dist/resources/collector.js +91 -9
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/model-fit/assess.d.ts +36 -0
- package/dist/resources/model-fit/assess.js +578 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
- package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
- package/dist/resources/model-fit/calibration/fit.js +152 -0
- package/dist/resources/model-fit/calibration/index.d.ts +37 -0
- package/dist/resources/model-fit/calibration/index.js +75 -0
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
- package/dist/resources/model-fit/estimators/llm.d.ts +45 -0
- package/dist/resources/model-fit/estimators/llm.js +220 -0
- package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
- package/dist/resources/model-fit/estimators/whisper.js +77 -0
- package/dist/resources/model-fit/types.d.ts +120 -0
- package/dist/resources/model-fit/types.js +1 -0
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/native.js +14 -0
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +16 -0
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +5 -7
- package/dist/runtime/model-registry.d.ts +0 -14
- package/dist/runtime/model-registry.js +17 -40
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +13 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +4 -5
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +18 -3
- package/dist/schemas/assess-model-fit.d.ts +271 -0
- package/dist/schemas/assess-model-fit.js +153 -0
- package/dist/schemas/audio-gen.d.ts +434 -4
- package/dist/schemas/audio-gen.js +131 -19
- package/dist/schemas/batch-completion-stream.d.ts +2 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.js +73 -27
- package/dist/schemas/bci.d.ts +2 -2
- package/dist/schemas/bci.js +5 -5
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/classification.js +13 -6
- package/dist/schemas/common.d.ts +662 -331
- package/dist/schemas/common.js +33 -31
- package/dist/schemas/completion-event.d.ts +2 -2
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +1 -5
- package/dist/schemas/completion-stream.js +4 -8
- package/dist/schemas/config.d.ts +640 -18
- package/dist/schemas/config.js +50 -8
- package/dist/schemas/download-asset.d.ts +19 -1
- package/dist/schemas/download-asset.js +20 -4
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +3 -3
- package/dist/schemas/error.d.ts +2 -2
- package/dist/schemas/error.js +2 -2
- package/dist/schemas/errors.d.ts +2 -6
- package/dist/schemas/errors.js +30 -40
- package/dist/schemas/heartbeat.d.ts +10 -0
- package/dist/schemas/heartbeat.js +8 -0
- package/dist/schemas/index.d.ts +46 -44
- package/dist/schemas/index.js +45 -43
- package/dist/schemas/llamacpp-config.d.ts +32 -10
- package/dist/schemas/llamacpp-config.js +11 -8
- package/dist/schemas/load-model.d.ts +2933 -1948
- package/dist/schemas/load-model.js +52 -52
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +2354 -0
- package/dist/schemas/model-config-schemas.js +46 -0
- package/dist/schemas/model-info.d.ts +10 -20
- package/dist/schemas/model-info.js +4 -25
- package/dist/schemas/model-ops.d.ts +0 -1
- package/dist/schemas/model-ops.js +1 -2
- package/dist/schemas/model-resource-profile.d.ts +109 -0
- package/dist/schemas/model-resource-profile.js +83 -0
- package/dist/schemas/model-src-utils.d.ts +1 -0
- package/dist/schemas/model-src-utils.js +33 -13
- package/dist/schemas/ocr.js +48 -13
- package/dist/schemas/path-validation.js +8 -2
- package/dist/schemas/plugin.d.ts +14 -1
- package/dist/schemas/plugin.js +11 -0
- package/dist/schemas/profiling.d.ts +138 -43
- package/dist/schemas/profiling.js +1 -22
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +0 -2
- package/dist/schemas/reload-config.js +2 -3
- package/dist/schemas/sdcpp-config.d.ts +344 -4
- package/dist/schemas/sdcpp-config.js +504 -17
- package/dist/schemas/system-resources.d.ts +138 -0
- package/dist/schemas/system-resources.js +3 -1
- package/dist/schemas/text-to-speech.js +183 -76
- package/dist/schemas/transcription-config.js +253 -85
- package/dist/schemas/transcription.d.ts +3 -3
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.js +53 -25
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/selection.d.ts +1 -5
- package/dist/selection.js +2 -6
- package/dist/surface.d.ts +1 -1
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +1 -1
- package/dist/utils/cache/auto-history.js +1 -1
- package/dist/utils/cache/paths.js +5 -5
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/image-dimensions.d.ts +26 -0
- package/dist/utils/image-dimensions.js +46 -0
- package/dist/utils/index.d.ts +13 -10
- package/dist/utils/index.js +13 -10
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-sanitize.d.ts +7 -0
- package/dist/utils/path-sanitize.js +36 -0
- package/dist/utils/path-security.d.ts +0 -7
- package/dist/utils/path-security.js +1 -36
- package/dist/utils/shard-utils.js +3 -3
- package/dist/utils/tool-integration.js +1 -1
- package/dist/utils/tools/dialect.js +1 -1
- package/dist/utils/tools/index.d.ts +2 -2
- package/dist/utils/tools/index.js +2 -2
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +3 -3
- package/dist/utils/url-security.d.ts +17 -0
- package/dist/utils/url-security.js +75 -0
- package/package.json +31 -26
- package/dist/api/provide.d.ts +0 -23
- package/dist/api/provide.js +0 -32
- package/dist/api/stop-provider.d.ts +0 -20
- package/dist/api/stop-provider.js +0 -30
- package/dist/p2p/cancel-delegated.d.ts +0 -3
- package/dist/p2p/cancel-delegated.js +0 -64
- package/dist/p2p/completion-stream-delegated.d.ts +0 -4
- package/dist/p2p/completion-stream-delegated.js +0 -51
- package/dist/p2p/delegate-client.d.ts +0 -9
- package/dist/p2p/delegate-client.js +0 -219
- package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
- package/dist/p2p/delegate-connect-diagnostics.js +0 -38
- package/dist/p2p/delegate-transport.d.ts +0 -18
- package/dist/p2p/delegate-transport.js +0 -237
- package/dist/p2p/error-handlers.d.ts +0 -4
- package/dist/p2p/error-handlers.js +0 -31
- package/dist/p2p/handle-request.d.ts +0 -2
- package/dist/p2p/handle-request.js +0 -171
- package/dist/p2p/handler-utils.d.ts +0 -19
- package/dist/p2p/handler-utils.js +0 -180
- package/dist/p2p/heartbeat-delegated.d.ts +0 -3
- package/dist/p2p/heartbeat-delegated.js +0 -32
- package/dist/p2p/load-model-delegated.d.ts +0 -6
- package/dist/p2p/load-model-delegated.js +0 -132
- package/dist/p2p/progress-throttle.d.ts +0 -7
- package/dist/p2p/progress-throttle.js +0 -47
- package/dist/p2p/provideHandler/connection.d.ts +0 -2
- package/dist/p2p/provideHandler/connection.js +0 -38
- package/dist/p2p/provideHandler/handler.d.ts +0 -2
- package/dist/p2p/provideHandler/handler.js +0 -66
- package/dist/p2p/provideHandler/proxy.d.ts +0 -2
- package/dist/p2p/provideHandler/proxy.js +0 -24
- package/dist/p2p/provideHandler.d.ts +0 -1
- package/dist/p2p/provideHandler.js +0 -2
- package/dist/p2p/rpc-utils.d.ts +0 -4
- package/dist/p2p/rpc-utils.js +0 -8
- package/dist/p2p/stopProvideHandler.d.ts +0 -2
- package/dist/p2p/stopProvideHandler.js +0 -29
- package/dist/p2p/swarm.d.ts +0 -9
- package/dist/p2p/swarm.js +0 -103
- package/dist/p2p/transport-selector.d.ts +0 -2
- package/dist/p2p/transport-selector.js +0 -18
- package/dist/p2p/unload-model-delegated.d.ts +0 -2
- package/dist/p2p/unload-model-delegated.js +0 -31
- package/dist/profiling/delegation-profiler.d.ts +0 -31
- package/dist/profiling/delegation-profiler.js +0 -155
- package/dist/schemas/delegate.d.ts +0 -29
- package/dist/schemas/delegate.js +0 -40
- package/dist/schemas/provide.d.ts +0 -50
- package/dist/schemas/provide.js +0 -32
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from '
|
|
2
|
-
import { extractSystemPrompt, getCurrentCacheInfo } from '
|
|
1
|
+
import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from './cache-logger.js';
|
|
2
|
+
import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
|
|
3
|
+
import { isAddonPreMutationRefusal, isAddonContextOverflowError } from './context-overflow.js';
|
|
3
4
|
import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
|
|
4
|
-
import { decideCachedHistorySlice, shouldCommitCachedTurn } from '
|
|
5
|
-
import { createKvCacheSession, generateConfigHash } from '
|
|
5
|
+
import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
|
|
6
|
+
import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
|
|
6
7
|
import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
7
8
|
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
8
9
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
@@ -11,7 +12,7 @@ import { getEngineLogger } from '../../../../logging/index.js';
|
|
|
11
12
|
import { AttachmentNotFoundError } from '../../../../errors/index.js';
|
|
12
13
|
import { nowMs } from '../../../../profiling/index.js';
|
|
13
14
|
import { buildStreamResult } from '../../../../profiling/model-execution.js';
|
|
14
|
-
import { normalizeCompletionStats, withEmittedTokens } from '
|
|
15
|
+
import { normalizeCompletionStats, withEmittedTokens } from './completion-stats.js';
|
|
15
16
|
import fs from 'bare-fs';
|
|
16
17
|
const logger = getEngineLogger();
|
|
17
18
|
function transformMessage(message) {
|
|
@@ -119,7 +120,7 @@ function rendersToolBlock(messages, toolBlock) {
|
|
|
119
120
|
* - The tool block travels only with the turn that writes it into the
|
|
120
121
|
* cache; see `skipToolBlock` below.
|
|
121
122
|
*/
|
|
122
|
-
function prepareMessagesForCache(session, turn, cacheExists, history, tools
|
|
123
|
+
function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
|
|
123
124
|
const toolBlock = tools?.length ? transformMessages(tools) : [];
|
|
124
125
|
if (!(cacheExists && history.length > 0)) {
|
|
125
126
|
const historyWithoutSystem = history.filter((msg) => msg.role !== 'system');
|
|
@@ -146,9 +147,8 @@ function prepareMessagesForCache(session, turn, cacheExists, history, tools, too
|
|
|
146
147
|
// conversation. Skip it only when the prefix is known to hold a rendered
|
|
147
148
|
// one: `toolBlockCached` records that a previous turn actually got it into
|
|
148
149
|
// the cache, which a committed message count does not prove. A stale
|
|
149
|
-
// boundary means we are resending the whole conversation anyway
|
|
150
|
-
|
|
151
|
-
const skipToolBlock = turn.toolBlockCached && !clearStaleCount && !toolBlockEvictable;
|
|
150
|
+
// boundary means we are resending the whole conversation anyway.
|
|
151
|
+
const skipToolBlock = turn.toolBlockCached && !clearStaleCount;
|
|
152
152
|
const blockToSend = skipToolBlock ? [] : toolBlock;
|
|
153
153
|
return {
|
|
154
154
|
messages: withToolBlock(transformMessages(messages), blockToSend),
|
|
@@ -208,12 +208,6 @@ export async function* completion(params, opts) {
|
|
|
208
208
|
const modelConfig = getModelConfig(modelId);
|
|
209
209
|
const toolsEnabled = modelConfig.tools === true;
|
|
210
210
|
const toolsActive = !!tools?.length && toolsEnabled;
|
|
211
|
-
// Sliding is opt-in (`n_discarded` defaults to 0). Once on, the addon's
|
|
212
|
-
// discard window opens at the end of the primed prefix — which is where the
|
|
213
|
-
// tool block sits, since the prime is the system prompt alone — and nothing
|
|
214
|
-
// protects it. So while sliding is possible the block cannot be assumed to
|
|
215
|
-
// survive, and it has to travel with every turn.
|
|
216
|
-
const toolBlockEvictable = (modelConfig.n_discarded ?? 0) > 0;
|
|
217
211
|
const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
|
|
218
212
|
// `responseFormat` is forwarded to the addon as a per-request
|
|
219
213
|
// `generationParams.json_schema`, which the addon converts to GBNF and
|
|
@@ -273,12 +267,11 @@ export async function* completion(params, opts) {
|
|
|
273
267
|
logMessagesToAddon(transformedHistory, 'NO_CACHE');
|
|
274
268
|
return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect, setActiveResponse);
|
|
275
269
|
}
|
|
276
|
-
// ---- KV-cache path. The session owns
|
|
277
|
-
//
|
|
278
|
-
//
|
|
279
|
-
// the happy path
|
|
280
|
-
//
|
|
281
|
-
// unwind through the same `scope.defer` hook. ----
|
|
270
|
+
// ---- KV-cache path. The session owns every bookkeeping layer; the
|
|
271
|
+
// handler registers one deferred unwind (`rollback`, or the non-destructive
|
|
272
|
+
// `releaseTurn` on a recognised pre-mutation refusal) that `commitTurn`
|
|
273
|
+
// short-circuits on the happy path. Cancellations / zero-token replies /
|
|
274
|
+
// rename failures all still unwind destructively through the same hook. ----
|
|
282
275
|
const session = createKvCacheSession(modelId, { logger: requestLogger });
|
|
283
276
|
const systemPromptFromHistory = extractSystemPrompt(history);
|
|
284
277
|
// The tool block is baked into the cache on the turn that first sends it and
|
|
@@ -319,16 +312,34 @@ export async function* completion(params, opts) {
|
|
|
319
312
|
// flips the turn's internal `committed` flag so this becomes a no-op
|
|
320
313
|
// on the happy path. Scope unwinding is LIFO — registered after the
|
|
321
314
|
// `removeEventListener` defer above so rollback runs before the
|
|
322
|
-
// listener detach.
|
|
323
|
-
|
|
315
|
+
// listener detach. A thrown overflow, pre-mutation refusal, or pre-addon
|
|
316
|
+
// attachment failure never persists the turn; the committed cache stays valid.
|
|
317
|
+
let preserveCacheOnUnwind = false;
|
|
318
|
+
scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
|
|
324
319
|
// `cacheExists` is implied by `beginTurn` — the session either found
|
|
325
320
|
// an existing cache or just primed one. Pass `true` to the message
|
|
326
321
|
// selector so the slicing branches engage.
|
|
327
|
-
|
|
328
|
-
|
|
322
|
+
let payload;
|
|
323
|
+
try {
|
|
324
|
+
payload = prepareMessagesForCache(session, turn,
|
|
325
|
+
/* cacheExists */ true, history, toolsActive ? tools : undefined);
|
|
326
|
+
}
|
|
327
|
+
catch (error) {
|
|
328
|
+
// A missing attachment is caller input rejected before the addon runs,
|
|
329
|
+
// so the committed cache is untouched and must survive.
|
|
330
|
+
preserveCacheOnUnwind = error instanceof AttachmentNotFoundError;
|
|
331
|
+
throw error;
|
|
332
|
+
}
|
|
329
333
|
const messagesToSend = payload.messages;
|
|
330
334
|
logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
|
|
331
|
-
|
|
335
|
+
let result;
|
|
336
|
+
try {
|
|
337
|
+
result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse);
|
|
338
|
+
}
|
|
339
|
+
catch (error) {
|
|
340
|
+
preserveCacheOnUnwind = isAddonContextOverflowError(error) || isAddonPreMutationRefusal(error);
|
|
341
|
+
throw error;
|
|
342
|
+
}
|
|
332
343
|
const shouldCommitTurn = shouldCommitCachedTurn({
|
|
333
344
|
aborted: signal.aborted,
|
|
334
345
|
producedTokens: result.producedTokens,
|
|
@@ -6,23 +6,24 @@
|
|
|
6
6
|
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
7
7
|
* `qvac_errors::StatusError::codeString()` and formats as
|
|
8
8
|
* `"[ <addonId> :: ContextOverflow ]"`. The message carries the
|
|
9
|
-
* C++-formatted detail (which may include the prompt/ctx sizes).
|
|
9
|
+
* C++-formatted detail (which may include the prompt/ctx sizes). That
|
|
10
|
+
* codeString survives only the synchronous throw path — asynchronous run
|
|
11
|
+
* errors are transported as `exception.what()` alone, with no code.
|
|
10
12
|
*
|
|
11
13
|
* These helpers let the plugin handler convert that addon error into a
|
|
12
14
|
* typed `ContextOverflowError` and let unit tests assert the detection
|
|
13
15
|
* + extraction logic without a real model load.
|
|
14
16
|
*/
|
|
15
17
|
export declare function isAddonContextOverflowError(err: unknown): boolean;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
* `MtmdLlmContext.cpp` format both numbers into the message; the
|
|
20
|
-
* `LlamaModel::processPromptImpl` fallback path emits a bare
|
|
21
|
-
* `"<func>: context overflow\n"` with neither number. Returning
|
|
22
|
-
* `undefined` for either field is fine — `ContextOverflowError` holds
|
|
23
|
-
* them as optional and the message factory degrades gracefully.
|
|
24
|
-
*/
|
|
25
|
-
export declare function parseContextOverflowMessage(message: string): {
|
|
18
|
+
export declare function isAddonPreMutationRefusal(err: unknown): boolean;
|
|
19
|
+
export type ContextOverflowSizes = {
|
|
20
|
+
/** The prompt alone, in tokens; unset when the guard reports KV cells. */
|
|
26
21
|
promptTokens?: number;
|
|
22
|
+
/** Cached conversation on a warm-cache guard, in `ctxSize` units. */
|
|
23
|
+
cachedTokens?: number;
|
|
24
|
+
/** Total the failing guard reported, in `ctxSize` units; the guards trigger on `>=`, so it can equal the window. */
|
|
25
|
+
requiredTokens?: number;
|
|
26
|
+
/** Effective per-request ceiling: `ctx_size` split across slots at `parallel > 1`. */
|
|
27
27
|
ctxSize?: number;
|
|
28
28
|
};
|
|
29
|
+
export declare function parseContextOverflowMessage(message: string): ContextOverflowSizes;
|
|
@@ -6,60 +6,152 @@
|
|
|
6
6
|
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
7
7
|
* `qvac_errors::StatusError::codeString()` and formats as
|
|
8
8
|
* `"[ <addonId> :: ContextOverflow ]"`. The message carries the
|
|
9
|
-
* C++-formatted detail (which may include the prompt/ctx sizes).
|
|
9
|
+
* C++-formatted detail (which may include the prompt/ctx sizes). That
|
|
10
|
+
* codeString survives only the synchronous throw path — asynchronous run
|
|
11
|
+
* errors are transported as `exception.what()` alone, with no code.
|
|
10
12
|
*
|
|
11
13
|
* These helpers let the plugin handler convert that addon error into a
|
|
12
14
|
* typed `ContextOverflowError` and let unit tests assert the detection
|
|
13
15
|
* + extraction logic without a real model load.
|
|
14
16
|
*/
|
|
17
|
+
// The guards' emitted starts; the numeric tails vary across addon
|
|
18
|
+
// generations, so the forms are start-anchored and single-line.
|
|
19
|
+
const CONTEXT_OVERFLOW_FORMS = [
|
|
20
|
+
/^\[TextLlm\] context overflow at (?:batch )?prefill step[^\r\n]*$/,
|
|
21
|
+
/^\[MtmdLlm\] context overflow at (?:batch )?prefill step[^\r\n]*$/,
|
|
22
|
+
// Unreachable at the pinned addon generation (generateResponse never
|
|
23
|
+
// reports !ok); kept in case a patch revives the path.
|
|
24
|
+
/^processPromptImpl: context overflow$/,
|
|
25
|
+
// The batch scheduler's capacity refusals are the same out-of-context
|
|
26
|
+
// condition, reported under a generic status.
|
|
27
|
+
/^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
28
|
+
/^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
29
|
+
/^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
30
|
+
/^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/
|
|
31
|
+
];
|
|
32
|
+
// Both detectors classify raw addon errors only — an already-typed QvacError
|
|
33
|
+
// (numeric `code`) is deliberately rejected by the present-code gate.
|
|
15
34
|
export function isAddonContextOverflowError(err) {
|
|
16
35
|
if (typeof err !== 'object' || err === null)
|
|
17
36
|
return false;
|
|
37
|
+
// A present status code is authoritative: exact ContextOverflow accepts,
|
|
38
|
+
// anything else (a non-string included) rejects with no message fallback.
|
|
18
39
|
const code = err.code;
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
// `ContextOverflowRecovered` or `PostContextOverflow` after an
|
|
22
|
-
// addon-side rename.
|
|
23
|
-
if (typeof code === 'string' && /::\s*ContextOverflow\s*\]/.test(code)) {
|
|
24
|
-
return true;
|
|
40
|
+
if (code !== undefined) {
|
|
41
|
+
return typeof code === 'string' && /^\[\s*[\w.-]+\s*::\s*ContextOverflow\s*\]$/.test(code);
|
|
25
42
|
}
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
// two known C++-emitted formats only — broader substring would catch
|
|
29
|
-
// wrapper / cause-chain text that mentions overflow without being one.
|
|
43
|
+
// The production path: the async transport strips status metadata from
|
|
44
|
+
// every run error, so the message alone identifies the overflow guards.
|
|
30
45
|
const message = err.message;
|
|
31
|
-
|
|
32
|
-
|
|
46
|
+
if (typeof message !== 'string')
|
|
47
|
+
return false;
|
|
48
|
+
const trimmed = message.trim();
|
|
49
|
+
return CONTEXT_OVERFLOW_FORMS.some((form) => form.test(trimmed));
|
|
33
50
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
*
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
// Refusals thrown before any decode or disk save; the async transport delivers
|
|
52
|
+
// exception.what() alone, so each form is matched complete and end-anchored.
|
|
53
|
+
const PRE_MUTATION_REFUSAL_FORMS = [
|
|
54
|
+
/^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
55
|
+
/^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
56
|
+
/^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
57
|
+
/^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
|
|
58
|
+
/^ContinuousBatchScheduler::submit: failed to add to batch \(MultiRequestBatcher::AddStatus=[1-4]\)$/,
|
|
59
|
+
/^invalid generationParams\.json_schema: [^\r\n]*$/,
|
|
60
|
+
/^failed to initialise sampler with per-request generationParams \(invalid grammar or json_schema\?\)$/,
|
|
61
|
+
/^\[MtmdLlm\] Media buffer is empty$/,
|
|
62
|
+
/^\[MtmdLlm\] Filename is empty$/,
|
|
63
|
+
/^\[MtmdLlm\] Failed to load media from file: [^\r\n]*$/,
|
|
64
|
+
/^\[MtmdLlm\] preparePrefill: prompt must end with text after the last media item$/
|
|
65
|
+
];
|
|
66
|
+
export function isAddonPreMutationRefusal(err) {
|
|
67
|
+
if (typeof err !== 'object' || err === null)
|
|
68
|
+
return false;
|
|
69
|
+
// A status code only exists on the synchronous throw path; when present it
|
|
70
|
+
// must be InvalidArgument, and its absence (the async transport) is fine.
|
|
71
|
+
const code = err.code;
|
|
72
|
+
if (code !== undefined &&
|
|
73
|
+
(typeof code !== 'string' || !/^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code))) {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
const message = err.message;
|
|
77
|
+
if (typeof message !== 'string')
|
|
78
|
+
return false;
|
|
79
|
+
const trimmed = message.trim();
|
|
80
|
+
return PRE_MUTATION_REFUSAL_FORMS.some((form) => form.test(trimmed));
|
|
81
|
+
}
|
|
82
|
+
// One entry per number-formatting guard, most specific first; keep in step
|
|
83
|
+
// with TextLlmContext.cpp / MtmdLlmContext.cpp or fields silently vanish.
|
|
84
|
+
const MESSAGE_PATTERNS = [
|
|
85
|
+
{
|
|
86
|
+
// "cached tokens C plus prompt tokens N exceed the max context tokens M"
|
|
87
|
+
pattern: /cached tokens (\d+) plus prompt tokens (\d+) exceeds? the max context tokens (\d+)/i,
|
|
88
|
+
map: ([cached, prompt, ctx]) => ({
|
|
89
|
+
promptTokens: prompt,
|
|
90
|
+
cachedTokens: cached,
|
|
91
|
+
requiredTokens: cached + prompt,
|
|
92
|
+
ctxSize: ctx
|
|
93
|
+
})
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
// "cached P positions / C KV cells plus ... of prompt exceed the max
|
|
97
|
+
// context tokens M" — KV cells, not tokens, so promptTokens stays unset.
|
|
98
|
+
pattern: /cached (\d+) positions \/ (\d+) KV cells plus (\d+) positions \/ (\d+) KV cells of prompt exceeds? the max context tokens (\d+)/i,
|
|
99
|
+
map: ([cPos, cCells, pPos, pCells, ctx]) => ({
|
|
100
|
+
cachedTokens: Math.max(cPos, cCells),
|
|
101
|
+
requiredTokens: Math.max(cPos + pPos, cCells + pCells),
|
|
102
|
+
ctxSize: ctx
|
|
103
|
+
})
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
// "prompt tokens N, max context tokens M"
|
|
107
|
+
pattern: /prompt tokens (\d+)[, \t]+max context tokens (\d+)/i,
|
|
108
|
+
map: ([prompt, ctx]) => ({ promptTokens: prompt, requiredTokens: prompt, ctxSize: ctx })
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
// "prompt spans P positions / N KV cells, max context tokens M" —
|
|
112
|
+
// not tokens, so promptTokens stays unset.
|
|
113
|
+
pattern: /prompt spans (\d+) positions \/ (\d+) KV cells,[ \t]*max context tokens (\d+)/i,
|
|
114
|
+
map: ([pos, cells, ctx]) => ({ requiredTokens: Math.max(pos, cells), ctxSize: ctx })
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
// "(N tokens, P positions, max M)" — the "tokens" figure is
|
|
118
|
+
// mtmd_helper_get_n_tokens, i.e. KV cells, so promptTokens stays unset.
|
|
119
|
+
pattern: /\((\d+)[ \t]+tokens,[ \t]*(\d+)[ \t]+positions,[ \t]*max[ \t]+(\d+)\)/i,
|
|
120
|
+
map: ([cells, pos, ctx]) => ({
|
|
121
|
+
requiredTokens: Math.max(cells, pos),
|
|
122
|
+
ctxSize: ctx
|
|
123
|
+
})
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
// "(N tokens, max M)", the retired short form. Both of its emitters
|
|
127
|
+
// format a cached total, not the prompt alone — promptTokens stays unset.
|
|
128
|
+
pattern: /\((\d+)[ \t]+tokens,[ \t]*max[ \t]+(\d+)\)/i,
|
|
129
|
+
map: ([total, ctx]) => ({ requiredTokens: total, ctxSize: ctx })
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
// Scheduler cap refusals: the total is cache-plus-prompt in cells or
|
|
133
|
+
// tokens, and the cap is the effective per-request ceiling.
|
|
134
|
+
pattern: /prompt of (\d+) (?:KV cells|tokens) (?:exceeds|leaves no room under) per-sequence cap (\d+)/,
|
|
135
|
+
map: ([total, cap]) => ({ requiredTokens: total, ctxSize: cap })
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
// "n_predict P + prompt N KV cells exceeds per-sequence cap M" — the
|
|
139
|
+
// reservation plus the prompt is the space the request needs.
|
|
140
|
+
pattern: /n_predict (\d+) \+ prompt (\d+) KV cells exceeds per-sequence cap (\d+)/,
|
|
141
|
+
map: ([predict, cells, cap]) => ({ requiredTokens: predict + cells, ctxSize: cap })
|
|
54
142
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
143
|
+
];
|
|
144
|
+
// Best-effort extraction of the overflow sizes; the bare processPromptImpl
|
|
145
|
+
// path carries no numbers, and undefined fields degrade gracefully.
|
|
146
|
+
export function parseContextOverflowMessage(message) {
|
|
147
|
+
for (const { pattern, map } of MESSAGE_PATTERNS) {
|
|
148
|
+
const match = message.match(pattern);
|
|
149
|
+
if (!match)
|
|
150
|
+
continue;
|
|
151
|
+
const numbers = match.slice(1).map(Number);
|
|
152
|
+
if (!numbers.every(Number.isFinite))
|
|
153
|
+
continue;
|
|
154
|
+
return map(numbers);
|
|
63
155
|
}
|
|
64
156
|
return {};
|
|
65
157
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { generateConfigHash } from '
|
|
1
|
+
import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
|
|
2
2
|
import { type CacheMessage } from '../../../../utils/index.js';
|
|
3
3
|
import type { Logger } from '../../../../logging/types.js';
|
|
4
4
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
@@ -106,6 +106,12 @@ export interface KvCacheSession {
|
|
|
106
106
|
* itself via `commitTurn`).
|
|
107
107
|
*/
|
|
108
108
|
rollback(turn: TurnHandle): Promise<void>;
|
|
109
|
+
/**
|
|
110
|
+
* Non-destructive counterpart of `rollback`: releases locks and refs but
|
|
111
|
+
* keeps the committed disk cache and its recorded prefix valid for a retry.
|
|
112
|
+
* A cache freshly primed by this same turn rolls back instead.
|
|
113
|
+
*/
|
|
114
|
+
releaseTurn(turn: TurnHandle): Promise<void>;
|
|
109
115
|
/**
|
|
110
116
|
* Forget the in-memory saved-message count for the turn's path
|
|
111
117
|
* without unlinking the file or clearing the init flag. Used when
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
|
-
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '
|
|
4
|
-
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '
|
|
3
|
+
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../ops/kv-cache-utils.js';
|
|
4
|
+
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../ops/kv-cache-retention.js';
|
|
5
5
|
import { isMobile } from '../../../../runtime/state.js';
|
|
6
6
|
import { getKVCacheDir } from '../../../../utils/index.js';
|
|
7
|
-
import { logCacheSaveError, logCacheStatus } from '
|
|
7
|
+
import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
|
|
8
8
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
9
9
|
// Used by cross-model paths that have no `RequestContext` (e.g.
|
|
10
10
|
// `deleteKvCacheState`). Per-session call sites receive a logger from
|
|
@@ -218,7 +218,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
218
218
|
...(signal !== undefined && { signal }),
|
|
219
219
|
releaseWriteLock,
|
|
220
220
|
committed: false,
|
|
221
|
-
rolledBack: false
|
|
221
|
+
rolledBack: false,
|
|
222
|
+
freshlyPrimed: false
|
|
222
223
|
});
|
|
223
224
|
markCachePathActive(cachePath);
|
|
224
225
|
return handle;
|
|
@@ -277,6 +278,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
277
278
|
await input.primeIfMissing(cachePath);
|
|
278
279
|
await verifyPrimedFile(cachePath, logger);
|
|
279
280
|
initializedCaches.add(cachePath);
|
|
281
|
+
turnState.get(handle).freshlyPrimed = true;
|
|
280
282
|
}
|
|
281
283
|
return handle;
|
|
282
284
|
}
|
|
@@ -362,6 +364,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
362
364
|
await input.primeIfMissing(cachePath);
|
|
363
365
|
await verifyPrimedFile(cachePath, logger);
|
|
364
366
|
initializedCaches.add(cachePath);
|
|
367
|
+
turnState.get(handle).freshlyPrimed = true;
|
|
365
368
|
}
|
|
366
369
|
return handle;
|
|
367
370
|
}
|
|
@@ -485,6 +488,24 @@ export function createKvCacheSession(modelId, options) {
|
|
|
485
488
|
return;
|
|
486
489
|
await runRollback(state);
|
|
487
490
|
}
|
|
491
|
+
async function releaseTurn(turn) {
|
|
492
|
+
const state = turnState.get(turn);
|
|
493
|
+
if (!state)
|
|
494
|
+
return;
|
|
495
|
+
if (state.committed || state.rolledBack)
|
|
496
|
+
return;
|
|
497
|
+
// A cache this same turn primed has no committed state to keep — a failed
|
|
498
|
+
// first turn must not leave its own prime behind.
|
|
499
|
+
if (state.freshlyPrimed) {
|
|
500
|
+
await runRollback(state);
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
releaseCachePath(state.cachePath);
|
|
504
|
+
state.rolledBack = true;
|
|
505
|
+
state.releaseWriteLock();
|
|
506
|
+
if (state.autoCacheKey !== undefined)
|
|
507
|
+
scheduleAutoCacheSweep(logger);
|
|
508
|
+
}
|
|
488
509
|
async function runRollback(state) {
|
|
489
510
|
// Order matters only weakly: unlink first so a partial disk-state
|
|
490
511
|
// can't be re-loaded by a sibling turn between the file delete and
|
|
@@ -520,6 +541,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
520
541
|
beginTurn,
|
|
521
542
|
commitTurn,
|
|
522
543
|
rollback,
|
|
544
|
+
releaseTurn,
|
|
523
545
|
dropStaleSavedCount
|
|
524
546
|
};
|
|
525
547
|
}
|
|
@@ -14,7 +14,13 @@ export declare const llmPlugin: {
|
|
|
14
14
|
device: import("zod").ZodOptional<import("zod").ZodString>;
|
|
15
15
|
predict: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodLiteral<-1>, import("zod").ZodLiteral<-2>, import("zod").ZodNumber]>>;
|
|
16
16
|
system_prompt: import("zod").ZodOptional<import("zod").ZodString>;
|
|
17
|
-
|
|
17
|
+
load_mode: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
18
|
+
none: "none";
|
|
19
|
+
mmap: "mmap";
|
|
20
|
+
mlock: "mlock";
|
|
21
|
+
"mmap+mlock": "mmap+mlock";
|
|
22
|
+
dio: "dio";
|
|
23
|
+
}>>;
|
|
18
24
|
verbosity: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
19
25
|
readonly ERROR: 0;
|
|
20
26
|
readonly WARN: 1;
|
|
@@ -25,7 +31,6 @@ export declare const llmPlugin: {
|
|
|
25
31
|
frequency_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
26
32
|
repeat_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
27
33
|
stop_sequences: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString>>;
|
|
28
|
-
n_discarded: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
29
34
|
parallel: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
30
35
|
tools: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
31
36
|
'cache-type-k': import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -38,6 +43,12 @@ export declare const llmPlugin: {
|
|
|
38
43
|
none: "none";
|
|
39
44
|
layer: "layer";
|
|
40
45
|
row: "row";
|
|
46
|
+
tensor: "tensor";
|
|
47
|
+
}>>;
|
|
48
|
+
'flash-attn': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
49
|
+
on: "on";
|
|
50
|
+
off: "off";
|
|
51
|
+
auto: "auto";
|
|
41
52
|
}>>;
|
|
42
53
|
'tensor-split': import("zod").ZodOptional<import("zod").ZodString>;
|
|
43
54
|
openclCacheDir: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -104,19 +115,19 @@ export declare const llmPlugin: {
|
|
|
104
115
|
device?: string | undefined;
|
|
105
116
|
predict?: number | undefined;
|
|
106
117
|
system_prompt?: string | undefined;
|
|
107
|
-
|
|
118
|
+
load_mode?: "none" | "mmap" | "mlock" | "mmap+mlock" | "dio" | undefined;
|
|
108
119
|
verbosity?: 0 | 3 | 2 | 1 | undefined;
|
|
109
120
|
presence_penalty?: number | undefined;
|
|
110
121
|
frequency_penalty?: number | undefined;
|
|
111
122
|
repeat_penalty?: number | undefined;
|
|
112
123
|
stop_sequences?: string[] | undefined;
|
|
113
|
-
n_discarded?: number | undefined;
|
|
114
124
|
parallel?: number | undefined;
|
|
115
125
|
tools?: boolean | undefined;
|
|
116
126
|
'cache-type-k'?: string | undefined;
|
|
117
127
|
'cache-type-v'?: string | undefined;
|
|
118
128
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
119
|
-
'split-mode'?: "none" | "layer" | "row" | undefined;
|
|
129
|
+
'split-mode'?: "none" | "layer" | "row" | "tensor" | undefined;
|
|
130
|
+
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
120
131
|
'tensor-split'?: string | undefined;
|
|
121
132
|
openclCacheDir?: string | undefined;
|
|
122
133
|
reasoning_budget?: number | undefined;
|
|
@@ -137,19 +148,19 @@ export declare const llmPlugin: {
|
|
|
137
148
|
device?: string | undefined;
|
|
138
149
|
predict?: number | undefined;
|
|
139
150
|
system_prompt?: string | undefined;
|
|
140
|
-
|
|
151
|
+
load_mode?: "none" | "mmap" | "mlock" | "mmap+mlock" | "dio" | undefined;
|
|
141
152
|
verbosity?: 0 | 3 | 2 | 1 | undefined;
|
|
142
153
|
presence_penalty?: number | undefined;
|
|
143
154
|
frequency_penalty?: number | undefined;
|
|
144
155
|
repeat_penalty?: number | undefined;
|
|
145
156
|
stop_sequences?: string[] | undefined;
|
|
146
|
-
n_discarded?: number | undefined;
|
|
147
157
|
parallel?: number | undefined;
|
|
148
158
|
tools?: boolean | undefined;
|
|
149
159
|
'cache-type-k'?: string | undefined;
|
|
150
160
|
'cache-type-v'?: string | undefined;
|
|
151
161
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
152
|
-
'split-mode'?: "none" | "layer" | "row" | undefined;
|
|
162
|
+
'split-mode'?: "none" | "layer" | "row" | "tensor" | undefined;
|
|
163
|
+
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
153
164
|
'tensor-split'?: string | undefined;
|
|
154
165
|
openclCacheDir?: string | undefined;
|
|
155
166
|
reasoning_budget?: number | undefined;
|
|
@@ -56,11 +56,11 @@ import { definePlugin, defineHandler, finetuneRequestSchema, batchCompletionStre
|
|
|
56
56
|
import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
|
|
57
57
|
import { getFirstShardPath } from '../../../utils/index.js';
|
|
58
58
|
import { getModelParallel } from '../../../utils/config-transform.js';
|
|
59
|
-
import { completion } from '
|
|
60
|
-
import { batchCompletion } from '
|
|
61
|
-
import { finetune } from '
|
|
62
|
-
import { translate } from '
|
|
63
|
-
import { transformLlmConfig } from '
|
|
59
|
+
import { completion } from './ops/completion-stream.js';
|
|
60
|
+
import { batchCompletion } from './ops/batch-completion-stream.js';
|
|
61
|
+
import { finetune } from './ops/finetune.js';
|
|
62
|
+
import { translate } from '../../ops/translate.js';
|
|
63
|
+
import { transformLlmConfig } from './transform.js';
|
|
64
64
|
import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
|
|
65
65
|
import { getModelConfig } from '../../../runtime/model-registry.js';
|
|
66
66
|
import { createCompletionNormalizer } from '../../../utils/completion-normalizer.js';
|
|
@@ -68,8 +68,8 @@ import { detectToolDialect } from '../../../utils/tool-integration.js';
|
|
|
68
68
|
import { getRequestRegistry, withRequestContext } from '../../../runtime/index.js';
|
|
69
69
|
import { generateRandomRequestId } from '../../../runtime/request-id.js';
|
|
70
70
|
import { ContextOverflowError } from '../../../errors/index.js';
|
|
71
|
-
import { isAddonContextOverflowError, parseContextOverflowMessage } from '
|
|
72
|
-
import { stoppedByLength } from '
|
|
71
|
+
import { isAddonContextOverflowError, parseContextOverflowMessage } from './ops/context-overflow.js';
|
|
72
|
+
import { stoppedByLength } from './ops/completion-stats.js';
|
|
73
73
|
import { isMobile } from '../../../runtime/state.js';
|
|
74
74
|
import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
|
|
75
75
|
function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
|
|
@@ -245,8 +245,7 @@ export const llmPlugin = definePlugin({
|
|
|
245
245
|
}
|
|
246
246
|
catch (err) {
|
|
247
247
|
if (isAddonContextOverflowError(err)) {
|
|
248
|
-
|
|
249
|
-
throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
|
|
248
|
+
throw new ContextOverflowError(parseContextOverflowMessage(err instanceof Error ? err.message : ''), request.modelId, err);
|
|
250
249
|
}
|
|
251
250
|
// Once the registry accepts cancellation, the request signal owns the
|
|
252
251
|
// terminal outcome. The addon may reject with different shapes
|
|
@@ -404,8 +403,7 @@ export const llmPlugin = definePlugin({
|
|
|
404
403
|
// signal, so a real overflow racing a cancel surfaces as a typed
|
|
405
404
|
// ContextOverflowError.
|
|
406
405
|
if (isAddonContextOverflowError(err)) {
|
|
407
|
-
|
|
408
|
-
throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
|
|
406
|
+
throw new ContextOverflowError(parseContextOverflowMessage(err instanceof Error ? err.message : ''), request.modelId, err);
|
|
409
407
|
}
|
|
410
408
|
// Context overflow is classified above. For every other error after
|
|
411
409
|
// accepted cancellation, the request signal owns the terminal outcome
|
|
@@ -483,8 +481,7 @@ export const llmPlugin = definePlugin({
|
|
|
483
481
|
// Same addon, same overflow path as `completionStream`. Wrap so
|
|
484
482
|
// translate consumers can `instanceof ContextOverflowError` too.
|
|
485
483
|
if (isAddonContextOverflowError(err)) {
|
|
486
|
-
|
|
487
|
-
throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
|
|
484
|
+
throw new ContextOverflowError(parseContextOverflowMessage(err instanceof Error ? err.message : ''), request.modelId, err);
|
|
488
485
|
}
|
|
489
486
|
throw err;
|
|
490
487
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { TurboVecIndexProvider } from '@qvac/rag';
|
|
1
2
|
import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
|
|
2
3
|
export declare const embeddingsPlugin: {
|
|
3
4
|
modelType: "llamacpp-embedding";
|
|
@@ -69,6 +70,9 @@ export declare const embeddingsPlugin: {
|
|
|
69
70
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
70
71
|
}, import("zod/v4/core").$strip>>;
|
|
71
72
|
};
|
|
73
|
+
capabilities: {
|
|
74
|
+
turbovecIndexProvider: TurboVecIndexProvider;
|
|
75
|
+
};
|
|
72
76
|
logging: {
|
|
73
77
|
module: import("@qvac/embed-llamacpp/addonLogging").AddonLogging;
|
|
74
78
|
namespace: "llamacpp-embedding";
|
|
@@ -1,12 +1,20 @@
|
|
|
1
|
-
import EmbedLlamacpp from '@qvac/embed-llamacpp';
|
|
1
|
+
import EmbedLlamacpp, { IdMapIndex } from '@qvac/embed-llamacpp';
|
|
2
2
|
import embedAddonLogging from '@qvac/embed-llamacpp/addonLogging';
|
|
3
3
|
import { definePlugin, defineHandler, embedRequestSchema, embedResponseSchema, ModelType, embedConfigBaseSchema, ADDON_EMBEDDING } from '../../../schemas/index.js';
|
|
4
4
|
import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
|
|
5
5
|
import { getFirstShardPath } from '../../../utils/index.js';
|
|
6
|
-
import { embed } from '
|
|
6
|
+
import { embed } from '../../ops/embed.js';
|
|
7
7
|
import { forwardModelExecution } from '../../../profiling/model-execution.js';
|
|
8
8
|
import { isMobile } from '../../../runtime/state.js';
|
|
9
9
|
import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
|
|
10
|
+
const turbovecIndexProvider = {
|
|
11
|
+
create(options) {
|
|
12
|
+
return new IdMapIndex(options);
|
|
13
|
+
},
|
|
14
|
+
load(snapshotPath) {
|
|
15
|
+
return IdMapIndex.load(snapshotPath);
|
|
16
|
+
}
|
|
17
|
+
};
|
|
10
18
|
function transformEmbedConfig(embedConfig) {
|
|
11
19
|
const config = {
|
|
12
20
|
device: embedConfig.device,
|
|
@@ -93,6 +101,9 @@ export const embeddingsPlugin = definePlugin({
|
|
|
93
101
|
}
|
|
94
102
|
})
|
|
95
103
|
},
|
|
104
|
+
capabilities: {
|
|
105
|
+
turbovecIndexProvider
|
|
106
|
+
},
|
|
96
107
|
logging: {
|
|
97
108
|
module: embedAddonLogging,
|
|
98
109
|
namespace: ModelType.llamacppEmbedding
|