@qvac/inference 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +89 -41
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +5 -2
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -2
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +39 -37
- package/dist/api/index.js +38 -36
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +6 -5
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +9 -0
- package/dist/errors/index.js +18 -1
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +84 -32
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -5
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +7 -5
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
- package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
- package/dist/model-fit/floor.d.ts +31 -0
- package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
- package/dist/model-fit/kv-cache.d.ts +23 -0
- package/dist/model-fit/kv-cache.js +85 -0
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
- package/dist/model-fit/platform.d.ts +14 -0
- package/dist/model-fit/platform.js +26 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
- package/dist/model-fit/request/create.js +61 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/model-fit/request/engines/audiogen.js +63 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
- package/dist/model-fit/request/engines/llama.js +44 -0
- package/dist/model-fit/request/engines/nmt.d.ts +8 -0
- package/dist/model-fit/request/engines/nmt.js +11 -0
- package/dist/model-fit/request/engines/ocr.d.ts +9 -0
- package/dist/model-fit/request/engines/ocr.js +13 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1072 -815
- package/dist/models/registry/models.js +3704 -3405
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +406 -3
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
- package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
- package/dist/plugins/builtin/index.d.ts +15 -12
- package/dist/plugins/builtin/index.js +15 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
- package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
- package/dist/plugins/ops/kv-cache-utils.js +25 -33
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +33 -19
- package/dist/plugins/ops/transcribe.js +18 -2
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +3 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +9 -1
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +22 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +7 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +106 -43
- package/dist/schemas/assess-model-fit.js +69 -32
- package/dist/schemas/audio-gen.d.ts +199 -19
- package/dist/schemas/audio-gen.js +183 -51
- package/dist/schemas/batch-completion-stream.d.ts +210 -36
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +3 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +24 -3
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +1465 -162
- package/dist/schemas/common.js +37 -31
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +243 -39
- package/dist/schemas/completion-stream.js +100 -29
- package/dist/schemas/config.d.ts +1326 -18
- package/dist/schemas/config.js +30 -8
- package/dist/schemas/decision.d.ts +610 -0
- package/dist/schemas/decision.js +219 -0
- package/dist/schemas/download-asset.d.ts +10 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +9 -1
- package/dist/schemas/engine-addon-map.js +26 -5
- package/dist/schemas/errors.d.ts +2 -0
- package/dist/schemas/errors.js +11 -1
- package/dist/schemas/index.d.ts +51 -48
- package/dist/schemas/index.js +50 -47
- package/dist/schemas/llamacpp-config.d.ts +222 -6
- package/dist/schemas/llamacpp-config.js +129 -7
- package/dist/schemas/load-model.d.ts +7960 -1492
- package/dist/schemas/load-model.js +122 -10
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +428 -9
- package/dist/schemas/model-config-schemas.js +16 -11
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +4 -3
- package/dist/schemas/model-resource-profile.d.ts +4 -4
- package/dist/schemas/model-resource-profile.js +2 -5
- package/dist/schemas/model-src-utils.d.ts +14 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +14 -1
- package/dist/schemas/model-types.js +4 -0
- package/dist/schemas/ocr.d.ts +76 -0
- package/dist/schemas/ocr.js +43 -1
- package/dist/schemas/plugin.d.ts +16 -4
- package/dist/schemas/plugin.js +12 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +34 -0
- package/dist/schemas/registry.js +5 -1
- package/dist/schemas/reload-config.d.ts +14 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +82 -0
- package/dist/schemas/sdcpp-config.js +41 -22
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +463 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +20 -0
- package/dist/schemas/transcription-config.js +54 -4
- package/dist/schemas/transcription.d.ts +47 -5
- package/dist/schemas/transcription.js +52 -5
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +40 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +3 -2
- package/dist/surface.js +3 -2
- package/dist/utils/addon-responses.d.ts +3 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +13 -9
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +2 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +68 -51
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -28
- package/dist/resources/model-fit/types.d.ts +0 -120
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
|
@@ -1,21 +1,26 @@
|
|
|
1
|
-
import { logCacheDisabled, logCacheSave, logMessagesToAddon } from '
|
|
2
|
-
import { extractSystemPrompt, getCurrentCacheInfo } from '
|
|
1
|
+
import { logCacheDisabled, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
|
|
2
|
+
import { extractSystemPrompt, getCurrentCacheInfo } from '../../../../plugins/ops/kv-cache-utils.js';
|
|
3
3
|
import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
|
|
4
|
-
import {
|
|
5
|
-
import { createKvCacheSession, generateConfigHash } from '
|
|
4
|
+
import { shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
|
|
5
|
+
import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
|
|
6
6
|
import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
7
|
-
import { resolveDeferredTools, toWireTool
|
|
7
|
+
import { resolveDeferredTools, toWireTool } from '../../../../utils/tools/defer.js';
|
|
8
8
|
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
9
9
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
10
|
-
import {
|
|
11
|
-
import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
|
|
10
|
+
import { buildAutoCacheSaveHistory, normalizeCacheMessage } from '../../../../utils/index.js';
|
|
12
11
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
13
12
|
import { AttachmentNotFoundError } from '../../../../errors/index.js';
|
|
14
13
|
import { nowMs } from '../../../../profiling/index.js';
|
|
15
14
|
import { buildStreamResult } from '../../../../profiling/model-execution.js';
|
|
16
|
-
import { normalizeCompletionStats, withEmittedTokens } from '
|
|
15
|
+
import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
|
|
17
16
|
import fs from 'bare-fs';
|
|
18
17
|
const logger = getEngineLogger();
|
|
18
|
+
export function withParallelToolCalls(generationParams) {
|
|
19
|
+
return {
|
|
20
|
+
...generationParams,
|
|
21
|
+
parallel_tool_calls: generationParams?.parallel_tool_calls ?? true
|
|
22
|
+
};
|
|
23
|
+
}
|
|
19
24
|
function transformMessage(message) {
|
|
20
25
|
const transformed = [];
|
|
21
26
|
// Check if it's a tool definition (has type: "function")
|
|
@@ -43,7 +48,16 @@ function transformMessage(message) {
|
|
|
43
48
|
}
|
|
44
49
|
transformed.push({
|
|
45
50
|
role: msg.role,
|
|
46
|
-
content: msg.content
|
|
51
|
+
content: msg.content,
|
|
52
|
+
...(msg.toolCalls && {
|
|
53
|
+
tool_calls: msg.toolCalls.map((call) => ({
|
|
54
|
+
...(call.id !== undefined && { id: call.id }),
|
|
55
|
+
name: call.name,
|
|
56
|
+
arguments: call.arguments
|
|
57
|
+
}))
|
|
58
|
+
}),
|
|
59
|
+
...(msg.toolCallId !== undefined && { tool_call_id: msg.toolCallId }),
|
|
60
|
+
...(msg.toolName !== undefined && { name: msg.toolName })
|
|
47
61
|
});
|
|
48
62
|
return transformed;
|
|
49
63
|
}
|
|
@@ -57,6 +71,18 @@ export function transformMessages(messages) {
|
|
|
57
71
|
}
|
|
58
72
|
return transformed;
|
|
59
73
|
}
|
|
74
|
+
// The fields that identify a turn, so two tool turns with equal `content`
|
|
75
|
+
// never share a cache key.
|
|
76
|
+
function toCacheMessage(msg) {
|
|
77
|
+
return {
|
|
78
|
+
role: msg.role,
|
|
79
|
+
content: msg.content,
|
|
80
|
+
attachments: msg.attachments ?? undefined,
|
|
81
|
+
...(msg.toolCalls && { toolCalls: msg.toolCalls }),
|
|
82
|
+
...(msg.toolCallId !== undefined && { toolCallId: msg.toolCallId }),
|
|
83
|
+
...(msg.toolName !== undefined && { toolName: msg.toolName })
|
|
84
|
+
};
|
|
85
|
+
}
|
|
60
86
|
/**
|
|
61
87
|
* Put the model's configured system prompt in front of a history that carries
|
|
62
88
|
* none. `transform.ts` strips `system_prompt` from the config handed to the
|
|
@@ -69,98 +95,54 @@ export function seedConfiguredSystemPrompt(history, modelConfig) {
|
|
|
69
95
|
// Every caller's message type makes everything past role and content optional.
|
|
70
96
|
return [{ role: 'system', content: configured }, ...history];
|
|
71
97
|
}
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
function withToolBlock(messages, toolBlock) {
|
|
78
|
-
if (toolBlock.length === 0)
|
|
79
|
-
return messages;
|
|
80
|
-
const systemIndex = messages.findIndex((msg) => msg.role === 'system');
|
|
81
|
-
if (systemIndex === -1)
|
|
82
|
-
return [...toolBlock, ...messages];
|
|
83
|
-
return [...messages.slice(0, systemIndex + 1), ...toolBlock, ...messages.slice(systemIndex + 1)];
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Fallback guess for whether a payload carrying this tool block gets it in
|
|
87
|
-
* front of the model, used only when the addon does not report
|
|
88
|
-
* `toolDefinitionsDropped`. Qwen-family templates anchor their tool section on
|
|
89
|
-
* the last user query and raise without one, and the addon answers that by
|
|
90
|
-
* re-rendering with tools stripped, so require a user message before believing
|
|
91
|
-
* the block landed.
|
|
92
|
-
*/
|
|
93
|
-
function rendersToolBlock(messages, toolBlock) {
|
|
94
|
-
if (toolBlock.length === 0)
|
|
98
|
+
// The addon refuses to save a key it holds nothing for — a cold turn it rolled
|
|
99
|
+
// back — with `InvalidArgument`. As with its other status errors, an
|
|
100
|
+
// asynchronous rejection carries the message alone.
|
|
101
|
+
function isNothingCachedRejection(error) {
|
|
102
|
+
if (typeof error !== 'object' || error === null)
|
|
95
103
|
return false;
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
* stand-in model) the payload's estimate stands.
|
|
103
|
-
*/
|
|
104
|
-
function resolveToolBlockCached(payload, stats) {
|
|
105
|
-
const dropped = stats?.toolDefinitionsDropped;
|
|
106
|
-
if (!payload.toolBlockSent || dropped === undefined)
|
|
107
|
-
return payload.toolBlockCached;
|
|
108
|
-
return payload.prefixHoldsBlock || (payload.toolBlockFull && dropped === 0);
|
|
104
|
+
const code = error.code;
|
|
105
|
+
if (code !== undefined) {
|
|
106
|
+
return typeof code === 'string' && /^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code);
|
|
107
|
+
}
|
|
108
|
+
const message = error.message;
|
|
109
|
+
return typeof message === 'string' && /no conversation is cached/.test(message);
|
|
109
110
|
}
|
|
110
111
|
/**
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
112
|
+
* Write the conversation the addon holds for `cachePath` to that file. The
|
|
113
|
+
* session commits on the file, and a saved conversation is one the addon drops
|
|
114
|
+
* once `deleteCache` removes the file. A cold turn the addon rolled back has
|
|
115
|
+
* nothing to write, and the commit then finds no file. Any other rejection is
|
|
116
|
+
* a write that failed: the file is behind the conversation, which the next
|
|
117
|
+
* turn re-decodes.
|
|
117
118
|
*/
|
|
118
|
-
async function
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
119
|
+
async function saveCacheFile(model, cachePath, log) {
|
|
120
|
+
const saveCache = model.saveCache;
|
|
121
|
+
if (!saveCache) {
|
|
122
|
+
log.debug(`[kv-cache] model has no saveCache; nothing written for ${cachePath}`);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
await saveCache.call(model, cachePath);
|
|
127
|
+
logCacheSave(cachePath);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
131
|
+
if (isNothingCachedRejection(error)) {
|
|
132
|
+
log.debug(`[kv-cache] saveCache had nothing to write for ${cachePath}: ${detail}`);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
log.warn(`[kv-cache] saveCache failed; the file is behind the conversation and the next turn re-decodes what it lacks. path=${cachePath} error=${detail}`);
|
|
132
136
|
}
|
|
133
|
-
// The block is never trimmed back out of the cache, so re-sending it every
|
|
134
|
-
// turn would leave one copy per turn and grow the prefix with the
|
|
135
|
-
// conversation. Skip it only when the prefix is known to hold a rendered
|
|
136
|
-
// one: `toolBlockCached` records that a previous turn actually got it into
|
|
137
|
-
// the cache, which a committed message count does not prove. A stale
|
|
138
|
-
// boundary means we are resending the whole conversation anyway.
|
|
139
|
-
const prefixHoldsBlock = turn.toolBlockCached && !clearStaleCount;
|
|
140
|
-
// The addon arms the tool-call grammar only for a payload that carries
|
|
141
|
-
// tools, and `required` / a named tool cannot be honoured without it. Such a
|
|
142
|
-
// turn resends the block even into a prefix that holds one; the second copy
|
|
143
|
-
// in the cache is the price of the guarantee.
|
|
144
|
-
const demandsCall = toolChoiceDemandsCall(toolChoice) && toolBlock.length > 0;
|
|
145
|
-
const skipToolBlock = prefixHoldsBlock && !demandsCall;
|
|
146
|
-
const blockToSend = skipToolBlock ? [] : toolBlock;
|
|
147
|
-
const toolBlockFull = blockToSend.length > 0 && (!demandsCall || toolChoice === 'required');
|
|
148
|
-
return {
|
|
149
|
-
messages: withToolBlock(transformMessages(messages), blockToSend),
|
|
150
|
-
toolBlockSent: blockToSend.length > 0,
|
|
151
|
-
toolBlockFull,
|
|
152
|
-
prefixHoldsBlock,
|
|
153
|
-
toolBlockCached: prefixHoldsBlock || (toolBlockFull && rendersToolBlock(messages, blockToSend))
|
|
154
|
-
};
|
|
155
137
|
}
|
|
156
|
-
async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse
|
|
138
|
+
async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse) {
|
|
157
139
|
const runOptions = {
|
|
158
140
|
...(generationParams && { generationParams }),
|
|
159
141
|
...(cacheOptions?.cacheKey !== undefined && {
|
|
160
142
|
cacheKey: cacheOptions.cacheKey
|
|
161
143
|
}),
|
|
162
|
-
...(cacheOptions?.
|
|
163
|
-
|
|
144
|
+
...(cacheOptions?.prefill !== undefined && {
|
|
145
|
+
prefill: cacheOptions.prefill
|
|
164
146
|
})
|
|
165
147
|
};
|
|
166
148
|
const hasRunOptions = Object.keys(runOptions).length > 0;
|
|
@@ -182,11 +164,6 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
|
|
|
182
164
|
yield { token: tokenStr };
|
|
183
165
|
}
|
|
184
166
|
const modelExecutionMs = nowMs() - modelStart;
|
|
185
|
-
// The addon has finished, and saved the cache file if it was going to.
|
|
186
|
-
onRunSettled?.();
|
|
187
|
-
if (cacheOptions?.saveCacheToDisk && cacheOptions.cacheKey) {
|
|
188
|
-
logCacheSave(cacheOptions.cacheKey);
|
|
189
|
-
}
|
|
190
167
|
if (tools && tools.length > 0) {
|
|
191
168
|
const { toolCalls } = parseToolCalls(accumulatedText, tools, dialect);
|
|
192
169
|
toolCallsResult = toolCalls;
|
|
@@ -199,24 +176,21 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
|
|
|
199
176
|
toolCalls: toolCallsResult,
|
|
200
177
|
responseText: accumulatedText,
|
|
201
178
|
producedTokens,
|
|
202
|
-
stoppedAtContextBoundary: stopReason === 'contextOverflow'
|
|
203
|
-
generationFinished: stopReason !== 'none'
|
|
179
|
+
stoppedAtContextBoundary: stopReason === 'contextOverflow'
|
|
204
180
|
};
|
|
205
181
|
}
|
|
206
182
|
export async function* completion(params, opts) {
|
|
207
|
-
const { history: requestHistory, modelId, kvCache, tools, generationParams, responseFormat } = params;
|
|
183
|
+
const { history: requestHistory, modelId, kvCache, prefill, tools, generationParams, responseFormat } = params;
|
|
208
184
|
const { signal, scope } = opts;
|
|
209
185
|
const requestLogger = opts.logger ?? logger;
|
|
210
186
|
const modelConfig = getModelConfig(modelId);
|
|
211
187
|
const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
|
|
212
188
|
const toolsEnabled = modelConfig.tools === true;
|
|
213
|
-
// With deferred tools the prompt
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
// history. `null` when nothing defers, which leaves the existing path alone.
|
|
189
|
+
// With deferred tools the prompt carries the always-loaded ones, `tool_search`
|
|
190
|
+
// and its catalog, and whatever earlier searches loaded. `null` when nothing
|
|
191
|
+
// defers, which leaves the existing path alone.
|
|
217
192
|
const deferred = resolveDeferredTools(tools, history);
|
|
218
193
|
const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
|
|
219
|
-
const callableTools = deferred?.callableTools ?? toolsToRender;
|
|
220
194
|
const toolsActive = !!toolsToRender?.length && toolsEnabled;
|
|
221
195
|
const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
|
|
222
196
|
// `responseFormat` is forwarded to the addon as a per-request
|
|
@@ -236,7 +210,7 @@ export async function* completion(params, opts) {
|
|
|
236
210
|
};
|
|
237
211
|
}
|
|
238
212
|
}
|
|
239
|
-
mergedGenerationParams =
|
|
213
|
+
mergedGenerationParams = withParallelToolCalls(mergedGenerationParams);
|
|
240
214
|
const model = getModel(modelId);
|
|
241
215
|
// Per-request hard cancel: under continuous batching the model runs several
|
|
242
216
|
// jobs at once, so an abort must cancel only THIS request's job, not the
|
|
@@ -267,30 +241,30 @@ export async function* completion(params, opts) {
|
|
|
267
241
|
// Drop the ref so a late abort can't cancel an already-settled response.
|
|
268
242
|
activeResponse = null;
|
|
269
243
|
});
|
|
244
|
+
// The cached and uncached paths send the same prompt: the whole conversation
|
|
245
|
+
// and the tools, every turn. With a cache key the addon keeps the longest
|
|
246
|
+
// prefix it already holds and decodes only the rest.
|
|
247
|
+
const prompt = transformMessages(toolsActive && toolsToRender ? prependToolsToHistory(history, toolsToRender) : history);
|
|
270
248
|
if (!kvCache) {
|
|
271
|
-
// KV-cache disabled — straight passthrough, no session involvement.
|
|
272
|
-
let historyWithTools = history;
|
|
273
|
-
if (toolsActive && toolsToRender) {
|
|
274
|
-
historyWithTools = prependToolsToHistory(history, toolsToRender);
|
|
275
|
-
}
|
|
276
|
-
const transformedHistory = transformMessages(historyWithTools);
|
|
277
249
|
logCacheDisabled();
|
|
278
|
-
logMessagesToAddon(
|
|
279
|
-
return yield* processModelResponse(model,
|
|
250
|
+
logMessagesToAddon(prompt, 'NO_CACHE');
|
|
251
|
+
return yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, undefined, dialect, setActiveResponse);
|
|
280
252
|
}
|
|
281
253
|
// ---- KV-cache path. The session owns every bookkeeping layer; the handler
|
|
282
|
-
// registers one deferred unwind that `commitTurn` short-circuits
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
//
|
|
287
|
-
//
|
|
288
|
-
const
|
|
289
|
-
const
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
254
|
+
// registers one deferred unwind that `commitTurn` short-circuits. The addon
|
|
255
|
+
// leaves the file consistent whatever the outcome — it commits or rolls back
|
|
256
|
+
// the request itself — so the unwind keeps it (`releaseTurn`) except where
|
|
257
|
+
// the file has no key to live under. A rollback also drops the addon's copy
|
|
258
|
+
// of the conversation, which a later key switch or unload would otherwise
|
|
259
|
+
// write back to the deleted path. ----
|
|
260
|
+
const discardCache = model.discardCache;
|
|
261
|
+
const session = createKvCacheSession(modelId, {
|
|
262
|
+
logger: requestLogger,
|
|
263
|
+
...(discardCache && {
|
|
264
|
+
discardCache: (cachePath) => discardCache.call(model, cachePath)
|
|
265
|
+
})
|
|
266
|
+
});
|
|
267
|
+
const configHash = generateConfigHash(extractSystemPrompt(history));
|
|
294
268
|
let turn;
|
|
295
269
|
if (typeof kvCache === 'string') {
|
|
296
270
|
turn = await session.beginTurn({
|
|
@@ -301,109 +275,72 @@ export async function* completion(params, opts) {
|
|
|
301
275
|
});
|
|
302
276
|
}
|
|
303
277
|
else {
|
|
304
|
-
const cacheMessages = history.map(
|
|
305
|
-
role: msg.role,
|
|
306
|
-
content: msg.content,
|
|
307
|
-
attachments: msg.attachments ?? undefined
|
|
308
|
-
}));
|
|
278
|
+
const cacheMessages = history.map(toCacheMessage);
|
|
309
279
|
turn = await session.beginTurn({
|
|
310
280
|
kind: 'auto',
|
|
311
281
|
configHash,
|
|
312
|
-
history: cacheMessages,
|
|
282
|
+
history: prefill ? cacheMessages.map(normalizeCacheMessage) : cacheMessages,
|
|
283
|
+
...(toolsActive && toolsToRender ? { tools: toolsToRender } : {}),
|
|
313
284
|
signal
|
|
314
285
|
});
|
|
315
286
|
}
|
|
316
|
-
//
|
|
317
|
-
//
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
// committed turn; `releaseTurn` still drops a cache this turn created.
|
|
323
|
-
let preserveCacheOnUnwind = false;
|
|
324
|
-
scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
|
|
325
|
-
let payload;
|
|
326
|
-
try {
|
|
327
|
-
payload = await prepareMessagesForCache(session, turn, history, toolsActive ? toolsToRender : undefined, mergedGenerationParams?.tool_choice);
|
|
328
|
-
}
|
|
329
|
-
catch (error) {
|
|
330
|
-
// A missing attachment is caller input rejected before the addon runs,
|
|
331
|
-
// so the committed cache is untouched and must survive.
|
|
332
|
-
preserveCacheOnUnwind = error instanceof AttachmentNotFoundError;
|
|
333
|
-
throw error;
|
|
334
|
-
}
|
|
335
|
-
const messagesToSend = payload.messages;
|
|
336
|
-
logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
|
|
337
|
-
let result;
|
|
338
|
-
let addonRunSettled = false;
|
|
339
|
-
try {
|
|
340
|
-
result = yield* processModelResponse(model, messagesToSend, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
|
|
341
|
-
addonRunSettled = true;
|
|
342
|
-
});
|
|
343
|
-
}
|
|
344
|
-
catch (error) {
|
|
345
|
-
// The addon writes the cache file only after a run completes and skips the
|
|
346
|
-
// save on every error path, so a run that threw left the committed file as
|
|
347
|
-
// it was. An engine-side throw after the run settled is the opposite: the
|
|
348
|
-
// file already holds this turn while no boundary was recorded for it.
|
|
349
|
-
preserveCacheOnUnwind = !addonRunSettled;
|
|
350
|
-
throw error;
|
|
351
|
-
}
|
|
352
|
-
const shouldCommitTurn = shouldCommitCachedTurn({
|
|
353
|
-
aborted: signal.aborted,
|
|
354
|
-
producedTokens: result.producedTokens,
|
|
355
|
-
generatedTokens: result.stats?.generatedTokens,
|
|
356
|
-
predict: mergedGenerationParams?.predict ?? modelConfig.predict,
|
|
357
|
-
stoppedAtContextBoundary: result.stoppedAtContextBoundary
|
|
358
|
-
});
|
|
359
|
-
// A cancelled run is rewound by the addon to the pre-request state before the
|
|
360
|
-
// file is re-saved, so the committed cache is intact. Every other non-commit
|
|
361
|
-
// finish (zero tokens, budget or context stop) was saved as-is and must go.
|
|
362
|
-
// An abort that landed after the addon reported a stop reason is the latter.
|
|
363
|
-
if (!shouldCommitTurn) {
|
|
364
|
-
preserveCacheOnUnwind = signal.aborted && !result.generationFinished;
|
|
365
|
-
}
|
|
287
|
+
// Scope unwinding is LIFO — registered after the `removeEventListener`
|
|
288
|
+
// defer above so this runs before the listener detach.
|
|
289
|
+
let rollbackOnUnwind = false;
|
|
290
|
+
scope.defer(() => (rollbackOnUnwind ? session.rollback(turn) : session.releaseTurn(turn)));
|
|
291
|
+
logMessagesToAddon(prompt, 'PROMPT_SEND');
|
|
292
|
+
const result = yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, { cacheKey: turn.cachePath, ...(prefill !== undefined && { prefill }) }, dialect, setActiveResponse);
|
|
366
293
|
if (typeof kvCache === 'string') {
|
|
367
|
-
// Custom-key path: the
|
|
368
|
-
//
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
await session.commitTurn(turn, {
|
|
372
|
-
kind: 'static',
|
|
373
|
-
messageCount: history.length + 1,
|
|
374
|
-
toolBlockCached: resolveToolBlockCached(payload, result.stats)
|
|
375
|
-
});
|
|
376
|
-
}
|
|
294
|
+
// Custom-key path: the file holds whatever the addon kept, cancelled and
|
|
295
|
+
// stopped turns included.
|
|
296
|
+
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
297
|
+
await session.commitTurn(turn, { kind: 'static' });
|
|
377
298
|
return result;
|
|
378
299
|
}
|
|
379
300
|
// Auto-cache path.
|
|
380
|
-
|
|
301
|
+
if (prefill && !signal.aborted && !result.stoppedAtContextBoundary) {
|
|
302
|
+
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
303
|
+
const cacheInfo = await getCurrentCacheInfo(modelId, configHash, history.map(toCacheMessage).map(normalizeCacheMessage), toolsActive ? toolsToRender : undefined);
|
|
304
|
+
await session.commitTurn(turn,
|
|
305
|
+
// Renaming to the held path would reacquire its non-reentrant write lock.
|
|
306
|
+
cacheInfo.cachePath === turn.cachePath
|
|
307
|
+
? { kind: 'static' }
|
|
308
|
+
: { kind: 'autoRename', targetCachePath: cacheInfo.cachePath });
|
|
309
|
+
return result;
|
|
310
|
+
}
|
|
381
311
|
// Tool-call turns: the auto-cache key is derived from
|
|
382
312
|
// `result.responseText`, which here is raw tool-call markup rather
|
|
383
313
|
// than a clean assistant message. There's no safe post-response key
|
|
384
|
-
// to rename to, so
|
|
314
|
+
// to rename to, so the deferred rollback drops the turn, unwritten. Once
|
|
385
315
|
// we support auto-cache for structured assistant/tool turns,
|
|
386
316
|
// this becomes a normal commit path.
|
|
387
317
|
if (result.toolCalls.length > 0) {
|
|
388
318
|
logger.warn(`[kv-cache] Auto cache tool-call turn; rolling back to avoid disk leak. path=${turn.cachePath}`);
|
|
319
|
+
rollbackOnUnwind = true;
|
|
389
320
|
return result;
|
|
390
321
|
}
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
322
|
+
const shouldRename = shouldCommitCachedTurn({
|
|
323
|
+
aborted: signal.aborted,
|
|
324
|
+
producedTokens: result.producedTokens,
|
|
325
|
+
generatedTokens: result.stats?.generatedTokens,
|
|
326
|
+
predict: mergedGenerationParams?.predict ?? modelConfig.predict,
|
|
327
|
+
stoppedAtContextBoundary: result.stoppedAtContextBoundary
|
|
328
|
+
});
|
|
329
|
+
if (!shouldRename) {
|
|
330
|
+
// A cancelled, empty or cut-off reply is not one the caller will send
|
|
331
|
+
// back as-is, so there is no post-response key to move to. The file stays
|
|
332
|
+
// under the history it was found by, holding what the addon kept. A first
|
|
333
|
+
// turn found no file, and one it wrote now would only be rolled back.
|
|
334
|
+
if (!turn.createdCache)
|
|
335
|
+
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
394
336
|
return result;
|
|
395
337
|
}
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
attachments: msg.attachments ?? undefined
|
|
400
|
-
})), result.responseText);
|
|
401
|
-
const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory);
|
|
338
|
+
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
339
|
+
const savedHistory = buildAutoCacheSaveHistory(history.map(toCacheMessage), result.responseText);
|
|
340
|
+
const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory, toolsActive ? toolsToRender : undefined);
|
|
402
341
|
await session.commitTurn(turn, {
|
|
403
342
|
kind: 'autoRename',
|
|
404
|
-
targetCachePath: postResponseCacheInfo.cachePath
|
|
405
|
-
messageCount: savedHistory.length,
|
|
406
|
-
toolBlockCached: resolveToolBlockCached(payload, result.stats)
|
|
343
|
+
targetCachePath: postResponseCacheInfo.cachePath
|
|
407
344
|
});
|
|
408
345
|
return result;
|
|
409
346
|
}
|