@qvac/inference 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +89 -41
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +5 -2
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -2
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +39 -37
- package/dist/api/index.js +38 -36
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +6 -5
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +9 -0
- package/dist/errors/index.js +18 -1
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +84 -32
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -5
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +7 -5
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
- package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
- package/dist/model-fit/floor.d.ts +31 -0
- package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
- package/dist/model-fit/kv-cache.d.ts +23 -0
- package/dist/model-fit/kv-cache.js +85 -0
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
- package/dist/model-fit/platform.d.ts +14 -0
- package/dist/model-fit/platform.js +26 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
- package/dist/model-fit/request/create.js +61 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/model-fit/request/engines/audiogen.js +63 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
- package/dist/model-fit/request/engines/llama.js +44 -0
- package/dist/model-fit/request/engines/nmt.d.ts +8 -0
- package/dist/model-fit/request/engines/nmt.js +11 -0
- package/dist/model-fit/request/engines/ocr.d.ts +9 -0
- package/dist/model-fit/request/engines/ocr.js +13 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1072 -815
- package/dist/models/registry/models.js +3704 -3405
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +406 -3
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
- package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
- package/dist/plugins/builtin/index.d.ts +15 -12
- package/dist/plugins/builtin/index.js +15 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
- package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
- package/dist/plugins/ops/kv-cache-utils.js +25 -33
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +33 -19
- package/dist/plugins/ops/transcribe.js +18 -2
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +3 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +9 -1
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +22 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +7 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +106 -43
- package/dist/schemas/assess-model-fit.js +69 -32
- package/dist/schemas/audio-gen.d.ts +199 -19
- package/dist/schemas/audio-gen.js +183 -51
- package/dist/schemas/batch-completion-stream.d.ts +210 -36
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +3 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +24 -3
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +1465 -162
- package/dist/schemas/common.js +37 -31
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +243 -39
- package/dist/schemas/completion-stream.js +100 -29
- package/dist/schemas/config.d.ts +1326 -18
- package/dist/schemas/config.js +30 -8
- package/dist/schemas/decision.d.ts +610 -0
- package/dist/schemas/decision.js +219 -0
- package/dist/schemas/download-asset.d.ts +10 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +9 -1
- package/dist/schemas/engine-addon-map.js +26 -5
- package/dist/schemas/errors.d.ts +2 -0
- package/dist/schemas/errors.js +11 -1
- package/dist/schemas/index.d.ts +51 -48
- package/dist/schemas/index.js +50 -47
- package/dist/schemas/llamacpp-config.d.ts +222 -6
- package/dist/schemas/llamacpp-config.js +129 -7
- package/dist/schemas/load-model.d.ts +7960 -1492
- package/dist/schemas/load-model.js +122 -10
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +428 -9
- package/dist/schemas/model-config-schemas.js +16 -11
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +4 -3
- package/dist/schemas/model-resource-profile.d.ts +4 -4
- package/dist/schemas/model-resource-profile.js +2 -5
- package/dist/schemas/model-src-utils.d.ts +14 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +14 -1
- package/dist/schemas/model-types.js +4 -0
- package/dist/schemas/ocr.d.ts +76 -0
- package/dist/schemas/ocr.js +43 -1
- package/dist/schemas/plugin.d.ts +16 -4
- package/dist/schemas/plugin.js +12 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +34 -0
- package/dist/schemas/registry.js +5 -1
- package/dist/schemas/reload-config.d.ts +14 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +82 -0
- package/dist/schemas/sdcpp-config.js +41 -22
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +463 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +20 -0
- package/dist/schemas/transcription-config.js +54 -4
- package/dist/schemas/transcription.d.ts +47 -5
- package/dist/schemas/transcription.js +52 -5
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +40 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +3 -2
- package/dist/surface.js +3 -2
- package/dist/utils/addon-responses.d.ts +3 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +13 -9
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +2 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +68 -51
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -28
- package/dist/resources/model-fit/types.d.ts +0 -120
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
|
@@ -1,27 +1,24 @@
|
|
|
1
1
|
import { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
|
-
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '
|
|
4
|
-
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '
|
|
3
|
+
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
|
|
4
|
+
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
|
|
5
5
|
import { isMobile } from '../../../../runtime/state.js';
|
|
6
6
|
import { getKVCacheDir } from '../../../../utils/index.js';
|
|
7
|
-
import { logCacheSaveError, logCacheStatus } from '
|
|
7
|
+
import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
|
|
8
8
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
9
|
-
import { z } from 'zod';
|
|
10
9
|
// Used by cross-model paths that have no `RequestContext` (e.g.
|
|
11
10
|
// `deleteKvCacheState`). Per-session call sites receive a logger from
|
|
12
11
|
// the caller — typically `withRequestContext(...)`.
|
|
13
12
|
const moduleLogger = getEngineLogger();
|
|
14
13
|
/**
|
|
15
|
-
* Coordinates
|
|
14
|
+
* Coordinates four KV-cache state layers:
|
|
16
15
|
*
|
|
17
|
-
* 1. `
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* 5. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
24
|
-
* 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
16
|
+
* 1. `initializedCaches` — caches this process has established.
|
|
17
|
+
* 2. On-disk `.bin` files written by the addon. Each one records the tokens
|
|
18
|
+
* it holds, and the addon reuses the longest prefix the next prompt
|
|
19
|
+
* shares with it, so nothing here tracks what a file covers.
|
|
20
|
+
* 3. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
21
|
+
* 4. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
25
22
|
*
|
|
26
23
|
* Every turn must finish through `commitTurn`, `rollback`, or the
|
|
27
24
|
* non-destructive `releaseTurn` so all inference state stays aligned,
|
|
@@ -30,100 +27,6 @@ const moduleLogger = getEngineLogger();
|
|
|
30
27
|
*/
|
|
31
28
|
// ----- module-scoped state. The session is the single mutation point
|
|
32
29
|
// for the in-memory KV-cache bookkeeping. -----
|
|
33
|
-
/** What the kv-cache file at a given path is known to hold. */
|
|
34
|
-
const cachedPrefixSchema = z.object({
|
|
35
|
-
/** Number of chat messages the file on disk is known to cover. */
|
|
36
|
-
messages: z.number().int().nonnegative(),
|
|
37
|
-
/**
|
|
38
|
-
* Whether a static tool block was rendered into that prefix. Tracked
|
|
39
|
-
* rather than inferred from `messages`, because a committed turn is not
|
|
40
|
-
* proof that its tool block reached the model: the addon drops tools and
|
|
41
|
-
* still returns a usable prompt when the chat template rejects them.
|
|
42
|
-
*/
|
|
43
|
-
toolBlock: z.boolean()
|
|
44
|
-
});
|
|
45
|
-
// The sidecar is written after the addon has already saved the `.bin`, so a
|
|
46
|
-
// crash between the two leaves a boundary that describes an older file. The
|
|
47
|
-
// `.bin` size is a function of the tokens it holds: it is unchanged when the
|
|
48
|
-
// addon re-saves the same state (cancel rewind, key switch) and larger once
|
|
49
|
-
// another turn landed in it, which is the case the fingerprint must catch.
|
|
50
|
-
const persistedPrefixSchema = cachedPrefixSchema.extend({
|
|
51
|
-
binSize: z.number().int().nonnegative()
|
|
52
|
-
});
|
|
53
|
-
/**
|
|
54
|
-
* What the kv-cache file on disk is known to cover, keyed by cache path.
|
|
55
|
-
* Written by `commitTurn`, deleted by `rollback`, `delete`,
|
|
56
|
-
* `dropStaleSavedCount` and the orphan clear in `beginTurn`. The same
|
|
57
|
-
* INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
|
|
58
|
-
* present only when the corresponding `.bin` file is considered trustworthy,
|
|
59
|
-
* so any turn that leaves the file holding something the boundary does not
|
|
60
|
-
* describe has to remove it.
|
|
61
|
-
*
|
|
62
|
-
* This map outranks the sidecar. Both are written by the same commit, so an
|
|
63
|
-
* entry here is never the older copy, and `restorePersistedCache` reads the
|
|
64
|
-
* file only to repopulate an empty map after a restart.
|
|
65
|
-
*/
|
|
66
|
-
const cachedPrefixes = new Map();
|
|
67
|
-
const PREFIX_SIDECAR_SUFFIX = '.meta.json';
|
|
68
|
-
function prefixSidecarPath(cachePath) {
|
|
69
|
-
return `${cachePath}${PREFIX_SIDECAR_SUFFIX}`;
|
|
70
|
-
}
|
|
71
|
-
async function writePrefixSidecar(cachePath, prefix, logger) {
|
|
72
|
-
try {
|
|
73
|
-
const { size } = await fsPromises.stat(cachePath);
|
|
74
|
-
await fsPromises.writeFile(prefixSidecarPath(cachePath), JSON.stringify({ ...prefix, binSize: size }));
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
// Only the file goes: a sidecar left from the previous commit describes a
|
|
78
|
-
// shorter `.bin` than the one now on disk. The in-memory boundary is the
|
|
79
|
-
// one this process keeps slicing against and stays.
|
|
80
|
-
await removePrefixSidecar(cachePath);
|
|
81
|
-
logger.warn(`[kv-cache] Failed to persist saved-message boundary; this process stays warm but a restart will start the cache cold. path=${cachePath} error=${error instanceof Error ? error.message : String(error)}`);
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
// Anything unreadable, malformed, or describing a `.bin` of a different size
|
|
85
|
-
// is discarded: the cache is then used from a cold boundary, which is the
|
|
86
|
-
// pre-sidecar behaviour. Only ever consulted for a path `cachedPrefixes` has
|
|
87
|
-
// no entry for, so it can never contradict a live boundary. Never throws: the
|
|
88
|
-
// caller holds the cache-path write lock and a `.bin` can vanish under it
|
|
89
|
-
// (`deleteKvCacheState` takes no locks).
|
|
90
|
-
async function readPrefixSidecar(cachePath) {
|
|
91
|
-
let parsed;
|
|
92
|
-
let binSize;
|
|
93
|
-
try {
|
|
94
|
-
const raw = await fsPromises.readFile(prefixSidecarPath(cachePath), 'utf8');
|
|
95
|
-
const result = persistedPrefixSchema.safeParse(JSON.parse(raw));
|
|
96
|
-
if (!result.success)
|
|
97
|
-
return null;
|
|
98
|
-
parsed = result.data;
|
|
99
|
-
binSize = (await fsPromises.stat(cachePath)).size;
|
|
100
|
-
}
|
|
101
|
-
catch {
|
|
102
|
-
return null;
|
|
103
|
-
}
|
|
104
|
-
if (binSize !== parsed.binSize) {
|
|
105
|
-
// The `.bin` changed after this boundary was recorded, so the file is
|
|
106
|
-
// stale whatever wrote it. Only the file: the caller guarantees the map
|
|
107
|
-
// holds nothing for this path.
|
|
108
|
-
await removePrefixSidecar(cachePath);
|
|
109
|
-
return null;
|
|
110
|
-
}
|
|
111
|
-
return { messages: parsed.messages, toolBlock: parsed.toolBlock };
|
|
112
|
-
}
|
|
113
|
-
/** Drops the persisted boundary, leaving whatever is in memory alone. */
|
|
114
|
-
async function removePrefixSidecar(cachePath) {
|
|
115
|
-
try {
|
|
116
|
-
await fsPromises.unlink(prefixSidecarPath(cachePath));
|
|
117
|
-
}
|
|
118
|
-
catch {
|
|
119
|
-
// No sidecar for this path.
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
/** Drops the boundary everywhere: the cache it describes is gone or untrusted. */
|
|
123
|
-
async function forgetPrefix(cachePath) {
|
|
124
|
-
cachedPrefixes.delete(cachePath);
|
|
125
|
-
await removePrefixSidecar(cachePath);
|
|
126
|
-
}
|
|
127
30
|
/**
|
|
128
31
|
* In-memory registry of caches established this session, recorded once a save
|
|
129
32
|
* is verified. The addon defers disk writes, so the absence of a `.bin` file on
|
|
@@ -154,7 +57,7 @@ class CacheLockAbortError extends Error {
|
|
|
154
57
|
// "session"), which name the SAME file on case-insensitive filesystems (default
|
|
155
58
|
// macOS/Windows), serialise on one lock and can't interleave writes. Over-locks
|
|
156
59
|
// case-only variants on case-sensitive filesystems, which is safe. NOTE: the
|
|
157
|
-
// other per-path bookkeeping (activeCachePaths,
|
|
60
|
+
// other per-path bookkeeping (activeCachePaths, initializedCaches, retention)
|
|
158
61
|
// stays case-sensitive, so case-only key variants are otherwise unsupported —
|
|
159
62
|
// see the KV-cache system doc.
|
|
160
63
|
function lockKeyFor(cachePath) {
|
|
@@ -335,20 +238,24 @@ async function deleteInactiveAutoCaches() {
|
|
|
335
238
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
336
239
|
* scope. `options.logger` is the per-instance logger the session emits
|
|
337
240
|
* through (typically `withRequestContext(getEngineLogger(), ctx)`);
|
|
338
|
-
* falls back to the module-scoped logger when omitted.
|
|
241
|
+
* falls back to the module-scoped logger when omitted. `options.discardCache`
|
|
242
|
+
* drops the addon's copy of a conversation whose cache the session rolls
|
|
243
|
+
* back; without it the addon writes that conversation to the deleted path at
|
|
244
|
+
* its next key switch or unload.
|
|
339
245
|
*/
|
|
340
246
|
export function createKvCacheSession(modelId, options) {
|
|
341
247
|
const logger = options?.logger ?? moduleLogger;
|
|
248
|
+
const discardCache = options?.discardCache;
|
|
342
249
|
// Per-session map: each `TurnHandle` carries an opaque entry here. A
|
|
343
250
|
// WeakMap so handles drop their state once the handler scope releases
|
|
344
251
|
// the reference; the module-scoped maps above survive.
|
|
345
252
|
const turnState = new WeakMap();
|
|
346
253
|
function makeHandle(cachePath, autoCacheKey, releaseWriteLock = () => { }, signal) {
|
|
347
|
-
const cached = cachedPrefixes.get(cachePath);
|
|
348
254
|
const handle = {
|
|
349
255
|
cachePath,
|
|
350
|
-
|
|
351
|
-
|
|
256
|
+
get createdCache() {
|
|
257
|
+
return turnState.get(handle)?.createdByThisTurn ?? false;
|
|
258
|
+
}
|
|
352
259
|
};
|
|
353
260
|
turnState.set(handle, {
|
|
354
261
|
cachePath,
|
|
@@ -389,14 +296,10 @@ export function createKvCacheSession(modelId, options) {
|
|
|
389
296
|
// In-memory registry check first — the addon defers disk writes, so a
|
|
390
297
|
// just-saved cache may not yet exist on disk. If the in-memory flag isn't
|
|
391
298
|
// set, fall back to a filesystem probe so caches surviving across process
|
|
392
|
-
// restarts still hit the reuse path.
|
|
393
|
-
// it snapshots the restored boundary.
|
|
299
|
+
// restarts still hit the reuse path.
|
|
394
300
|
let exists = initializedCaches.has(cachePath);
|
|
395
301
|
if (!exists)
|
|
396
|
-
exists = await
|
|
397
|
-
// A boundary without its `.bin` describes a cache that no longer exists.
|
|
398
|
-
if (!exists)
|
|
399
|
-
await forgetPrefix(cachePath);
|
|
302
|
+
exists = await adoptPersistedCache(cachePath);
|
|
400
303
|
const handle = makeHandle(cachePath, undefined, releaseWriteLock, input.signal);
|
|
401
304
|
try {
|
|
402
305
|
logCacheStatus(input.customKey, exists);
|
|
@@ -420,8 +323,9 @@ export function createKvCacheSession(modelId, options) {
|
|
|
420
323
|
// the state lock, or commitTurn's rename could deadlock a waiter), then
|
|
421
324
|
// re-check existence under the lock in case a same-file peer just renamed it.
|
|
422
325
|
const discovered = await withCacheStateLock(async () => {
|
|
423
|
-
const existingCache = await findMatchingCache(modelId, input.configHash, input.history);
|
|
424
|
-
const cacheInfo = existingCache ??
|
|
326
|
+
const existingCache = await findMatchingCache(modelId, input.configHash, input.history, input.tools);
|
|
327
|
+
const cacheInfo = existingCache ??
|
|
328
|
+
(await getCurrentCacheInfo(modelId, input.configHash, input.history, input.tools));
|
|
425
329
|
return {
|
|
426
330
|
cachePath: cacheInfo.cachePath,
|
|
427
331
|
cacheKey: cacheInfo.cacheKey
|
|
@@ -511,33 +415,31 @@ export function createKvCacheSession(modelId, options) {
|
|
|
511
415
|
if (state.committed || state.rolledBack)
|
|
512
416
|
return;
|
|
513
417
|
if (result.kind === 'static') {
|
|
514
|
-
//
|
|
515
|
-
|
|
516
|
-
// currently swallows save errors — see TODO in
|
|
517
|
-
// `verifySaveAndRecord`) and record the new boundary.
|
|
518
|
-
const prefix = { messages: result.messageCount, toolBlock: result.toolBlockCached };
|
|
519
|
-
const ok = await verifySaveAndRecord(state.cachePath, prefix);
|
|
418
|
+
// The addon's save wrote the new cache state at the same path.
|
|
419
|
+
const ok = await verifySaveAndRecord(state.cachePath);
|
|
520
420
|
if (!ok) {
|
|
521
421
|
await runRollback(state);
|
|
522
422
|
return;
|
|
523
423
|
}
|
|
524
|
-
await writePrefixSidecar(state.cachePath, prefix, logger);
|
|
525
424
|
state.committed = true;
|
|
526
425
|
releaseCachePath(state.cachePath);
|
|
527
426
|
state.releaseWriteLock();
|
|
427
|
+
if (state.autoCacheKey !== undefined)
|
|
428
|
+
scheduleAutoCacheSweep(logger);
|
|
528
429
|
return;
|
|
529
430
|
}
|
|
530
431
|
const sourceCachePath = state.cachePath;
|
|
531
432
|
const sourceCacheKey = state.autoCacheKey;
|
|
532
433
|
const targetCacheKey = path.basename(path.dirname(path.dirname(result.targetCachePath)));
|
|
533
434
|
// Hold the target's lock across the rename AND commit verification, so a peer
|
|
534
|
-
// resolving to the same file can't observe it before
|
|
535
|
-
// published. Abortable: a cancel while waiting for the lock rejects here.
|
|
435
|
+
// resolving to the same file can't observe it before it is recorded. Abortable: a cancel while waiting for the lock rejects here.
|
|
536
436
|
const releaseTargetLock = await acquireCachePathWriteLock(result.targetCachePath, state.signal);
|
|
537
437
|
try {
|
|
538
|
-
// A cancel that landed while we waited for the target lock must not
|
|
438
|
+
// A cancel that landed while we waited for the target lock must not
|
|
439
|
+
// commit. The file holds what the addon kept, so it stays unless this
|
|
440
|
+
// turn created it.
|
|
539
441
|
if (state.signal?.aborted) {
|
|
540
|
-
await
|
|
442
|
+
await runRelease(state);
|
|
541
443
|
return;
|
|
542
444
|
}
|
|
543
445
|
const renamed = await withCacheStateLock(async () => {
|
|
@@ -564,7 +466,6 @@ export function createKvCacheSession(modelId, options) {
|
|
|
564
466
|
releaseCachePath(sourceCachePath);
|
|
565
467
|
state.cachePath = result.targetCachePath;
|
|
566
468
|
state.autoCacheKey = targetCacheKey;
|
|
567
|
-
await forgetPrefix(sourceCachePath);
|
|
568
469
|
await pruneEmptyCacheDirectories(sourceCachePath, snapshotActivePaths());
|
|
569
470
|
if (sourceCacheKey !== undefined) {
|
|
570
471
|
await removeAutoCacheMarkerIfMissing(sourceCacheKey);
|
|
@@ -579,10 +480,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
579
480
|
await runRollback(state);
|
|
580
481
|
return;
|
|
581
482
|
}
|
|
582
|
-
const ok = await verifySaveAndRecord(result.targetCachePath
|
|
583
|
-
messages: result.messageCount,
|
|
584
|
-
toolBlock: result.toolBlockCached
|
|
585
|
-
});
|
|
483
|
+
const ok = await verifySaveAndRecord(result.targetCachePath);
|
|
586
484
|
if (!ok) {
|
|
587
485
|
// Rename succeeded but the file isn't where we expected. Roll back via
|
|
588
486
|
// the target path instead of the (now-empty) source.
|
|
@@ -615,6 +513,9 @@ export function createKvCacheSession(modelId, options) {
|
|
|
615
513
|
return;
|
|
616
514
|
if (state.committed || state.rolledBack)
|
|
617
515
|
return;
|
|
516
|
+
await runRelease(state);
|
|
517
|
+
}
|
|
518
|
+
async function runRelease(state) {
|
|
618
519
|
// Nothing committed exists to keep, so a failed first turn is destructive.
|
|
619
520
|
if (state.createdByThisTurn) {
|
|
620
521
|
await runRollback(state);
|
|
@@ -641,7 +542,14 @@ export function createKvCacheSession(modelId, options) {
|
|
|
641
542
|
logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
|
|
642
543
|
}
|
|
643
544
|
}
|
|
644
|
-
|
|
545
|
+
if (discardCache) {
|
|
546
|
+
try {
|
|
547
|
+
await discardCache(state.cachePath);
|
|
548
|
+
}
|
|
549
|
+
catch (discardError) {
|
|
550
|
+
logger.warn(`[kv-cache] Failed to discard the addon's copy of a rolled-back cache; a later unload may write it back. path=${state.cachePath} error=${discardError instanceof Error ? discardError.message : String(discardError)}`);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
645
553
|
// Release before pruning so an empty parent can go; a sibling still holding
|
|
646
554
|
// the path keeps it in the active snapshot and protects the directory.
|
|
647
555
|
releaseCachePath(state.cachePath);
|
|
@@ -655,18 +563,11 @@ export function createKvCacheSession(modelId, options) {
|
|
|
655
563
|
if (state.autoCacheKey !== undefined)
|
|
656
564
|
scheduleAutoCacheSweep(logger);
|
|
657
565
|
}
|
|
658
|
-
async function dropStaleSavedCount(turn) {
|
|
659
|
-
const state = turnState.get(turn);
|
|
660
|
-
if (!state)
|
|
661
|
-
return;
|
|
662
|
-
await forgetPrefix(state.cachePath);
|
|
663
|
-
}
|
|
664
566
|
return {
|
|
665
567
|
beginTurn,
|
|
666
568
|
commitTurn,
|
|
667
569
|
rollback,
|
|
668
|
-
releaseTurn
|
|
669
|
-
dropStaleSavedCount
|
|
570
|
+
releaseTurn
|
|
670
571
|
};
|
|
671
572
|
}
|
|
672
573
|
// ----- module-level administrative API -----
|
|
@@ -687,10 +588,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
687
588
|
* Layers cleared, in order:
|
|
688
589
|
* 1. On-disk: `deleteCache(...)` removes the matching directory
|
|
689
590
|
* tree (or wipes and recreates the root for `all: true`).
|
|
690
|
-
* 2. `
|
|
691
|
-
*
|
|
692
|
-
* 3. `initializedCaches`: prefix-cleanup by the same removed
|
|
693
|
-
* directory, since it is keyed by the resolved cache path.
|
|
591
|
+
* 2. `initializedCaches`: prefix-cleanup by the removed directory,
|
|
592
|
+
* since it is keyed by the resolved cache path.
|
|
694
593
|
*
|
|
695
594
|
* Concurrency with in-flight turns: this delete does not take the
|
|
696
595
|
* per-cache-path write locks, so it races any turn holding a `TurnHandle`
|
|
@@ -699,9 +598,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
699
598
|
* `Map.delete` / `Set.delete` no-op on absent keys), but the layers are
|
|
700
599
|
* cleared without a lock, so an interleaving that lands the delete's
|
|
701
600
|
* in-memory cleanup after a concurrent turn has already renamed/committed
|
|
702
|
-
* its file splits state: the file stays on disk while its
|
|
703
|
-
*
|
|
704
|
-
* `savedCount=0`. Callers must not delete a key that is in active use.
|
|
601
|
+
* its file splits state: the file stays on disk while its init flag is
|
|
602
|
+
* cleared. Callers must not delete a key that is in active use.
|
|
705
603
|
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
706
604
|
* and clears each one under the cache-state lock.
|
|
707
605
|
*/
|
|
@@ -710,7 +608,6 @@ export async function deleteKvCacheState(target) {
|
|
|
710
608
|
return deleteInactiveAutoCaches();
|
|
711
609
|
if ('all' in target) {
|
|
712
610
|
const removed = await deleteCacheUtil({ all: true });
|
|
713
|
-
cachedPrefixes.clear();
|
|
714
611
|
initializedCaches.clear();
|
|
715
612
|
// `removed` is the kv-cache root dir; surfaces it for ops
|
|
716
613
|
// visibility but isn't part of the contract.
|
|
@@ -728,50 +625,40 @@ export async function deleteKvCacheState(target) {
|
|
|
728
625
|
// onto the unrelated top-level auto marker.
|
|
729
626
|
await removeAutoCacheMarker(path.relative(getKVCacheDir(), removedPath));
|
|
730
627
|
}
|
|
731
|
-
//
|
|
732
|
-
//
|
|
733
|
-
//
|
|
734
|
-
// or sits beneath it must go.
|
|
735
|
-
clearCachedMessageCountsByPrefix(removedPath, path.sep);
|
|
736
|
-
// initializedCaches is keyed by the resolved cachePath too, so clear it by
|
|
737
|
-
// the same removed-directory prefix as cachedPrefixes above.
|
|
628
|
+
// The on-disk directory tree is `{kvCacheRoot}/{kvCacheKey}[/{modelId}]/`,
|
|
629
|
+
// so every entry whose path is the removed directory itself or sits beneath
|
|
630
|
+
// it must go.
|
|
738
631
|
clearInitializedCachesByPrefix(removedPath, path.sep);
|
|
739
632
|
}
|
|
740
633
|
// ----- private helpers -----
|
|
741
634
|
/**
|
|
742
|
-
* Verify the
|
|
743
|
-
*
|
|
744
|
-
*
|
|
745
|
-
* history rather than slicing against a stale `savedCount`.
|
|
746
|
-
*
|
|
747
|
-
* TODO: once the addon surfaces save failures (e.g. throws
|
|
748
|
-
* `UnableToSaveSessionFile` when `llama_state_save_file` returns
|
|
749
|
-
* false), drop the `access()` probe and wrap the `model.run()` call in
|
|
750
|
-
* a real try/catch that forwards the error.
|
|
635
|
+
* Verify the cache file exists before recording it as established. The file
|
|
636
|
+
* is the evidence because `saveCache` rejects both when a write fails and when
|
|
637
|
+
* the addon kept nothing to write.
|
|
751
638
|
*/
|
|
752
|
-
async function verifySaveAndRecord(cachePath
|
|
639
|
+
async function verifySaveAndRecord(cachePath) {
|
|
753
640
|
try {
|
|
754
641
|
await fsPromises.access(cachePath);
|
|
755
|
-
cachedPrefixes.set(cachePath, prefix);
|
|
756
642
|
initializedCaches.add(cachePath);
|
|
757
643
|
return true;
|
|
758
644
|
}
|
|
759
645
|
catch (err) {
|
|
760
|
-
cachedPrefixes.delete(cachePath);
|
|
761
646
|
logCacheSaveError(cachePath, err);
|
|
762
647
|
return false;
|
|
763
648
|
}
|
|
764
649
|
}
|
|
765
650
|
/**
|
|
766
|
-
* Adopt a `.bin` left by an earlier process run
|
|
767
|
-
*
|
|
768
|
-
*
|
|
769
|
-
* An auto-rename commits its target into `cachedPrefixes` without an init
|
|
770
|
-
* flag, so this can run for a path the map already knows. The map wins there
|
|
771
|
-
* and the sidecar is left unread, which keeps the two from ever having to be
|
|
772
|
-
* reconciled.
|
|
651
|
+
* Adopt a `.bin` left by an earlier process run. Returns whether the file
|
|
652
|
+
* exists. Earlier releases kept a `<bin>.meta.json` boundary sidecar beside
|
|
653
|
+
* it; nothing reads one now, so it goes on first contact.
|
|
773
654
|
*/
|
|
774
|
-
async function
|
|
655
|
+
async function adoptPersistedCache(cachePath) {
|
|
656
|
+
try {
|
|
657
|
+
await fsPromises.unlink(`${cachePath}.meta.json`);
|
|
658
|
+
}
|
|
659
|
+
catch {
|
|
660
|
+
// No sidecar for this path.
|
|
661
|
+
}
|
|
775
662
|
try {
|
|
776
663
|
await fsPromises.access(cachePath);
|
|
777
664
|
}
|
|
@@ -779,28 +666,8 @@ async function restorePersistedCache(cachePath) {
|
|
|
779
666
|
return false;
|
|
780
667
|
}
|
|
781
668
|
initializedCaches.add(cachePath);
|
|
782
|
-
if (cachedPrefixes.has(cachePath))
|
|
783
|
-
return true;
|
|
784
|
-
const prefix = await readPrefixSidecar(cachePath);
|
|
785
|
-
if (prefix !== null)
|
|
786
|
-
cachedPrefixes.set(cachePath, prefix);
|
|
787
669
|
return true;
|
|
788
670
|
}
|
|
789
|
-
function clearCachedMessageCountsByPrefix(prefix, sep) {
|
|
790
|
-
if (!prefix) {
|
|
791
|
-
cachedPrefixes.clear();
|
|
792
|
-
return;
|
|
793
|
-
}
|
|
794
|
-
for (const key of cachedPrefixes.keys()) {
|
|
795
|
-
if (key === prefix) {
|
|
796
|
-
cachedPrefixes.delete(key);
|
|
797
|
-
continue;
|
|
798
|
-
}
|
|
799
|
-
if (!key.startsWith(prefix + sep))
|
|
800
|
-
continue;
|
|
801
|
-
cachedPrefixes.delete(key);
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
671
|
function clearInitializedCachesByPrefix(prefix, sep) {
|
|
805
672
|
if (!prefix) {
|
|
806
673
|
initializedCaches.clear();
|
|
@@ -822,21 +689,6 @@ function clearInitializedCachesByPrefix(prefix, sep) {
|
|
|
822
689
|
* @internal
|
|
823
690
|
*/
|
|
824
691
|
export const __kvCacheSessionTestHooks = {
|
|
825
|
-
getSavedCount(cachePath) {
|
|
826
|
-
return cachedPrefixes.get(cachePath)?.messages;
|
|
827
|
-
},
|
|
828
|
-
setSavedCountForTest(cachePath, count) {
|
|
829
|
-
cachedPrefixes.set(cachePath, { messages: count, toolBlock: false });
|
|
830
|
-
},
|
|
831
|
-
getToolBlockCachedForTest(cachePath) {
|
|
832
|
-
return cachedPrefixes.get(cachePath)?.toolBlock ?? false;
|
|
833
|
-
},
|
|
834
|
-
getPrefixSidecarPathForTest(cachePath) {
|
|
835
|
-
return prefixSidecarPath(cachePath);
|
|
836
|
-
},
|
|
837
|
-
readPrefixSidecarForTest(cachePath) {
|
|
838
|
-
return readPrefixSidecar(cachePath);
|
|
839
|
-
},
|
|
840
692
|
hasInitializedPath(cachePath) {
|
|
841
693
|
return initializedCaches.has(cachePath);
|
|
842
694
|
},
|
|
@@ -850,7 +702,6 @@ export const __kvCacheSessionTestHooks = {
|
|
|
850
702
|
lastAutoCacheSweepMs = value;
|
|
851
703
|
},
|
|
852
704
|
resetForTest() {
|
|
853
|
-
cachedPrefixes.clear();
|
|
854
705
|
initializedCaches.clear();
|
|
855
706
|
activeCachePaths.clear();
|
|
856
707
|
lastAutoCacheSweepMs = 0;
|
|
@@ -1,37 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pure decision
|
|
3
|
-
* `completion-stream.ts` (via the slice computed from
|
|
4
|
-
* `TurnHandle.savedCount`).
|
|
2
|
+
* Pure commit decision for kv-cache turns used by `completion-stream.ts`.
|
|
5
3
|
*
|
|
6
4
|
* This module intentionally has **no** `bare-*` imports so it can be
|
|
7
|
-
* exercised directly from unit tests running under
|
|
5
|
+
* exercised directly from unit tests running under Node without
|
|
8
6
|
* pulling in the Bare runtime (which is not available in that
|
|
9
7
|
* environment).
|
|
10
|
-
*
|
|
11
|
-
* Cancel detection flows through the per-request `AbortSignal` from
|
|
12
|
-
* `RequestRegistry`; the `cachedMessageCounts` map lives in
|
|
13
|
-
* `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
|
|
14
|
-
* layers (saved counts, init flags, on-disk files). Only the pure
|
|
15
|
-
* slice-decision helper remains here.
|
|
16
8
|
*/
|
|
17
|
-
export interface HistoryMessage {
|
|
18
|
-
role: string;
|
|
19
|
-
content: string;
|
|
20
|
-
attachments?: {
|
|
21
|
-
path: string;
|
|
22
|
-
}[] | undefined;
|
|
23
|
-
}
|
|
24
|
-
export interface HistorySliceDecision {
|
|
25
|
-
/** Messages to send to the model on the next turn. */
|
|
26
|
-
messages: HistoryMessage[];
|
|
27
|
-
/**
|
|
28
|
-
* True when the decision path proves the current `savedCount` is stale
|
|
29
|
-
* and the caller should drop the cached entry (via
|
|
30
|
-
* `KvCacheSession.dropStaleSavedCount(turn)`) to avoid propagating the
|
|
31
|
-
* bad count to the next turn.
|
|
32
|
-
*/
|
|
33
|
-
clearStaleCount: boolean;
|
|
34
|
-
}
|
|
35
9
|
interface CacheCommitContext {
|
|
36
10
|
aborted: boolean;
|
|
37
11
|
producedTokens: boolean;
|
|
@@ -40,25 +14,4 @@ interface CacheCommitContext {
|
|
|
40
14
|
stoppedAtContextBoundary: boolean;
|
|
41
15
|
}
|
|
42
16
|
export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
|
|
43
|
-
/**
|
|
44
|
-
* Pure slice decision for `prepareMessagesForCache`.
|
|
45
|
-
*
|
|
46
|
-
* Mirrors the shape of the logic in `completion-stream.ts` but without
|
|
47
|
-
* calling `transformMessages` (which depends on `bare-fs` for
|
|
48
|
-
* attachment probing). Kept here so the decision can be unit-tested in
|
|
49
|
-
* isolation.
|
|
50
|
-
*
|
|
51
|
-
* A non-zero `savedCount` is itself the proof that a cache holds a rendered
|
|
52
|
-
* prefix: it is recorded only after a save is verified. It means the prefix
|
|
53
|
-
* covers that many messages, so slicing from it drops exactly what the cache
|
|
54
|
-
* supplies — the system message among them. Without a usable boundary the
|
|
55
|
-
* cache supplies nothing, so the whole history goes to the addon, system
|
|
56
|
-
* message included: anything held back would simply be missing from the
|
|
57
|
-
* prompt.
|
|
58
|
-
*
|
|
59
|
-
* The regression guard: a non-zero `savedCount` that slices the history down
|
|
60
|
-
* to an empty array is stale, and the caller resends the full history rather
|
|
61
|
-
* than handing the model an empty payload.
|
|
62
|
-
*/
|
|
63
|
-
export declare function decideCachedHistorySlice(savedCount: number, history: HistoryMessage[]): HistorySliceDecision;
|
|
64
17
|
export {};
|
|
@@ -1,18 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pure decision
|
|
3
|
-
* `completion-stream.ts` (via the slice computed from
|
|
4
|
-
* `TurnHandle.savedCount`).
|
|
2
|
+
* Pure commit decision for kv-cache turns used by `completion-stream.ts`.
|
|
5
3
|
*
|
|
6
4
|
* This module intentionally has **no** `bare-*` imports so it can be
|
|
7
|
-
* exercised directly from unit tests running under
|
|
5
|
+
* exercised directly from unit tests running under Node without
|
|
8
6
|
* pulling in the Bare runtime (which is not available in that
|
|
9
7
|
* environment).
|
|
10
|
-
*
|
|
11
|
-
* Cancel detection flows through the per-request `AbortSignal` from
|
|
12
|
-
* `RequestRegistry`; the `cachedMessageCounts` map lives in
|
|
13
|
-
* `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
|
|
14
|
-
* layers (saved counts, init flags, on-disk files). Only the pure
|
|
15
|
-
* slice-decision helper remains here.
|
|
16
8
|
*/
|
|
17
9
|
export function shouldCommitCachedTurn(context) {
|
|
18
10
|
const { aborted, producedTokens, generatedTokens, predict, stoppedAtContextBoundary } = context;
|
|
@@ -22,39 +14,3 @@ export function shouldCommitCachedTurn(context) {
|
|
|
22
14
|
generatedTokens >= predict;
|
|
23
15
|
return !aborted && producedTokens && !stoppedByBudget && !stoppedAtContextBoundary;
|
|
24
16
|
}
|
|
25
|
-
/**
|
|
26
|
-
* Pure slice decision for `prepareMessagesForCache`.
|
|
27
|
-
*
|
|
28
|
-
* Mirrors the shape of the logic in `completion-stream.ts` but without
|
|
29
|
-
* calling `transformMessages` (which depends on `bare-fs` for
|
|
30
|
-
* attachment probing). Kept here so the decision can be unit-tested in
|
|
31
|
-
* isolation.
|
|
32
|
-
*
|
|
33
|
-
* A non-zero `savedCount` is itself the proof that a cache holds a rendered
|
|
34
|
-
* prefix: it is recorded only after a save is verified. It means the prefix
|
|
35
|
-
* covers that many messages, so slicing from it drops exactly what the cache
|
|
36
|
-
* supplies — the system message among them. Without a usable boundary the
|
|
37
|
-
* cache supplies nothing, so the whole history goes to the addon, system
|
|
38
|
-
* message included: anything held back would simply be missing from the
|
|
39
|
-
* prompt.
|
|
40
|
-
*
|
|
41
|
-
* The regression guard: a non-zero `savedCount` that slices the history down
|
|
42
|
-
* to an empty array is stale, and the caller resends the full history rather
|
|
43
|
-
* than handing the model an empty payload.
|
|
44
|
-
*/
|
|
45
|
-
export function decideCachedHistorySlice(savedCount, history) {
|
|
46
|
-
const hasCachedPrefix = savedCount > 0 && history.length > 0;
|
|
47
|
-
const sliced = hasCachedPrefix && savedCount <= history.length ? history.slice(savedCount) : null;
|
|
48
|
-
// A non-null slice that is empty means the saved count is stale: the
|
|
49
|
-
// cached turn boundary is claiming the entire current history is
|
|
50
|
-
// already cached, which happens when a previous turn was cancelled
|
|
51
|
-
// mid-decode and still recorded `history.length + 1`. Treat it as a
|
|
52
|
-
// bad state and resend the full history.
|
|
53
|
-
const useSlice = sliced !== null && sliced.length > 0;
|
|
54
|
-
return {
|
|
55
|
-
messages: useSlice ? sliced : history,
|
|
56
|
-
// Only a boundary that was actually consulted can be stale. Absent a
|
|
57
|
-
// history to slice, the count is simply unused and left alone.
|
|
58
|
-
clearStaleCount: hasCachedPrefix && !useSlice
|
|
59
|
-
};
|
|
60
|
-
}
|