@qvac/inference 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +89 -41
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +5 -2
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -2
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +39 -37
- package/dist/api/index.js +38 -36
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +6 -5
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +9 -0
- package/dist/errors/index.js +18 -1
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +84 -32
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -5
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +7 -5
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
- package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
- package/dist/model-fit/floor.d.ts +31 -0
- package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
- package/dist/model-fit/kv-cache.d.ts +23 -0
- package/dist/model-fit/kv-cache.js +85 -0
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
- package/dist/model-fit/platform.d.ts +14 -0
- package/dist/model-fit/platform.js +26 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
- package/dist/model-fit/request/create.js +61 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/model-fit/request/engines/audiogen.js +63 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
- package/dist/model-fit/request/engines/llama.js +44 -0
- package/dist/model-fit/request/engines/nmt.d.ts +8 -0
- package/dist/model-fit/request/engines/nmt.js +11 -0
- package/dist/model-fit/request/engines/ocr.d.ts +9 -0
- package/dist/model-fit/request/engines/ocr.js +13 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1072 -815
- package/dist/models/registry/models.js +3704 -3405
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +406 -3
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
- package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
- package/dist/plugins/builtin/index.d.ts +15 -12
- package/dist/plugins/builtin/index.js +15 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
- package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
- package/dist/plugins/ops/kv-cache-utils.js +25 -33
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +33 -19
- package/dist/plugins/ops/transcribe.js +18 -2
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +3 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +9 -1
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +22 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +7 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +106 -43
- package/dist/schemas/assess-model-fit.js +69 -32
- package/dist/schemas/audio-gen.d.ts +199 -19
- package/dist/schemas/audio-gen.js +183 -51
- package/dist/schemas/batch-completion-stream.d.ts +210 -36
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +3 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +24 -3
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +1465 -162
- package/dist/schemas/common.js +37 -31
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +243 -39
- package/dist/schemas/completion-stream.js +100 -29
- package/dist/schemas/config.d.ts +1326 -18
- package/dist/schemas/config.js +30 -8
- package/dist/schemas/decision.d.ts +610 -0
- package/dist/schemas/decision.js +219 -0
- package/dist/schemas/download-asset.d.ts +10 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +9 -1
- package/dist/schemas/engine-addon-map.js +26 -5
- package/dist/schemas/errors.d.ts +2 -0
- package/dist/schemas/errors.js +11 -1
- package/dist/schemas/index.d.ts +51 -48
- package/dist/schemas/index.js +50 -47
- package/dist/schemas/llamacpp-config.d.ts +222 -6
- package/dist/schemas/llamacpp-config.js +129 -7
- package/dist/schemas/load-model.d.ts +7960 -1492
- package/dist/schemas/load-model.js +122 -10
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +428 -9
- package/dist/schemas/model-config-schemas.js +16 -11
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +4 -3
- package/dist/schemas/model-resource-profile.d.ts +4 -4
- package/dist/schemas/model-resource-profile.js +2 -5
- package/dist/schemas/model-src-utils.d.ts +14 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +14 -1
- package/dist/schemas/model-types.js +4 -0
- package/dist/schemas/ocr.d.ts +76 -0
- package/dist/schemas/ocr.js +43 -1
- package/dist/schemas/plugin.d.ts +16 -4
- package/dist/schemas/plugin.js +12 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +34 -0
- package/dist/schemas/registry.js +5 -1
- package/dist/schemas/reload-config.d.ts +14 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +82 -0
- package/dist/schemas/sdcpp-config.js +41 -22
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +463 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +20 -0
- package/dist/schemas/transcription-config.js +54 -4
- package/dist/schemas/transcription.d.ts +47 -5
- package/dist/schemas/transcription.js +52 -5
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +40 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +3 -2
- package/dist/surface.js +3 -2
- package/dist/utils/addon-responses.d.ts +3 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +13 -9
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +2 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +68 -51
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -28
- package/dist/resources/model-fit/types.d.ts +0 -120
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
|
@@ -1,52 +1,22 @@
|
|
|
1
|
-
import { generateConfigHash } from '
|
|
1
|
+
import { generateConfigHash } from '../../../../plugins/ops/kv-cache-utils.js';
|
|
2
2
|
import { type CacheMessage } from '../../../../utils/index.js';
|
|
3
|
+
import type { Tool } from '../../../../schemas/tools.js';
|
|
3
4
|
import type { Logger } from '../../../../logging/types.js';
|
|
4
5
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
5
|
-
import { z } from 'zod';
|
|
6
|
-
/**
|
|
7
|
-
* Coordinates six KV-cache state layers:
|
|
8
|
-
*
|
|
9
|
-
* 1. `cachedPrefixes` — saved message boundaries, and whether a tool
|
|
10
|
-
* block was rendered into them.
|
|
11
|
-
* 2. `initializedCaches` — caches this process has established.
|
|
12
|
-
* 3. On-disk `.bin` files written by the addon.
|
|
13
|
-
* 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
|
|
14
|
-
* boundary outlives the process; see `readPrefixSidecar`.
|
|
15
|
-
* 5. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
16
|
-
* 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
17
|
-
*
|
|
18
|
-
* Every turn must finish through `commitTurn`, `rollback`, or the
|
|
19
|
-
* non-destructive `releaseTurn` so all inference state stays aligned,
|
|
20
|
-
* the active-path ref is released, and marker metadata follows the
|
|
21
|
-
* cache directory lifecycle.
|
|
22
|
-
*/
|
|
23
|
-
/** What the kv-cache file at a given path is known to hold. */
|
|
24
|
-
declare const cachedPrefixSchema: z.ZodObject<{
|
|
25
|
-
messages: z.ZodNumber;
|
|
26
|
-
toolBlock: z.ZodBoolean;
|
|
27
|
-
}, z.core.$strip>;
|
|
28
|
-
type CachedPrefix = z.infer<typeof cachedPrefixSchema>;
|
|
29
6
|
export interface TurnHandle {
|
|
30
7
|
/** Resolved on-disk cache file path the addon will read from / write to. */
|
|
31
8
|
readonly cachePath: string;
|
|
32
9
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* message tail for the next addon call.
|
|
36
|
-
*/
|
|
37
|
-
readonly savedCount: number;
|
|
38
|
-
/**
|
|
39
|
-
* Whether the cached prefix already holds a rendered static tool block, so
|
|
40
|
-
* this turn can leave it out of its payload. False on a cold turn and
|
|
41
|
-
* whenever the previous turn couldn't confirm the block reached the model.
|
|
10
|
+
* True when no cache existed at this path when the turn began: nothing
|
|
11
|
+
* committed is there to keep, so an unwind drops whatever this turn wrote.
|
|
42
12
|
*/
|
|
43
|
-
readonly
|
|
13
|
+
readonly createdCache: boolean;
|
|
44
14
|
}
|
|
45
15
|
export interface BeginCustomTurnInput {
|
|
46
16
|
kind: 'custom';
|
|
47
17
|
/** User-provided session key (`completion({ kvCache: "session-a" })`). */
|
|
48
18
|
customKey: string;
|
|
49
|
-
/** Hash of system prompt
|
|
19
|
+
/** Hash of the system prompt; see `generateConfigHash`. */
|
|
50
20
|
configHash: string;
|
|
51
21
|
/**
|
|
52
22
|
* Request abort signal. When it aborts while this turn is queued behind a
|
|
@@ -57,38 +27,25 @@ export interface BeginCustomTurnInput {
|
|
|
57
27
|
}
|
|
58
28
|
export interface BeginAutoTurnInput {
|
|
59
29
|
kind: 'auto';
|
|
60
|
-
/** Hash of system prompt
|
|
30
|
+
/** Hash of the system prompt; see `generateConfigHash`. */
|
|
61
31
|
configHash: string;
|
|
62
32
|
/** Conversation history used to compute the pre-response cache key. */
|
|
63
33
|
history: CacheMessage[];
|
|
34
|
+
/** Rendered tools that distinguish prefixes with no assistant message. */
|
|
35
|
+
tools?: readonly Tool[];
|
|
64
36
|
/** See `BeginCustomTurnInput.signal`. */
|
|
65
37
|
signal?: AbortSignal;
|
|
66
38
|
}
|
|
67
39
|
export type BeginTurnInput = BeginCustomTurnInput | BeginAutoTurnInput;
|
|
68
|
-
|
|
69
|
-
* Whether the committed prefix holds a rendered static tool block. The
|
|
70
|
-
* handler owns this because only it knows whether the payload carried the
|
|
71
|
-
* block and whether the message list was one the template renders tools for.
|
|
72
|
-
*/
|
|
73
|
-
interface ToolBlockCommit {
|
|
74
|
-
toolBlockCached: boolean;
|
|
75
|
-
}
|
|
76
|
-
export interface StaticCommitResult extends ToolBlockCommit {
|
|
40
|
+
export interface StaticCommitResult {
|
|
77
41
|
kind: 'static';
|
|
78
|
-
/** `history.length + 1` — recorded at the turn's current `cachePath`. */
|
|
79
|
-
messageCount: number;
|
|
80
42
|
}
|
|
81
|
-
export interface AutoRenameCommitResult
|
|
43
|
+
export interface AutoRenameCommitResult {
|
|
82
44
|
kind: 'autoRename';
|
|
83
45
|
/**
|
|
84
|
-
* Destination path the
|
|
85
|
-
* renamed to (computed from `cacheMessages + responseText`). The
|
|
86
|
-
* stale entry at the source path is dropped from `cachedPrefixes`
|
|
87
|
-
* and the new count is recorded at this target path.
|
|
46
|
+
* Destination path for the history covered by the saved cache.
|
|
88
47
|
*/
|
|
89
48
|
targetCachePath: string;
|
|
90
|
-
/** Number of messages the renamed cache represents (`savedHistory.length`). */
|
|
91
|
-
messageCount: number;
|
|
92
49
|
}
|
|
93
50
|
export type CommitResult = StaticCommitResult | AutoRenameCommitResult;
|
|
94
51
|
export interface KvCacheSession {
|
|
@@ -96,53 +53,48 @@ export interface KvCacheSession {
|
|
|
96
53
|
* Open a new turn against the cache. Resolves the cache file path,
|
|
97
54
|
* ensures its parent directory exists, and returns a `TurnHandle` the
|
|
98
55
|
* handler attaches to `ctx.scope.defer(...)` for the rollback hook. The
|
|
99
|
-
* addon's
|
|
100
|
-
*
|
|
56
|
+
* addon's `saveCache` writes the file after the run, so a cold turn opens
|
|
57
|
+
* without one. Auto-cache path resolution is serialized with
|
|
101
58
|
* retention deletion before the handle is returned.
|
|
102
59
|
*/
|
|
103
60
|
beginTurn(input: BeginTurnInput): Promise<TurnHandle>;
|
|
104
61
|
/**
|
|
105
|
-
* Commit a
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
* a no-op on the happy path.
|
|
62
|
+
* Commit a turn — verifies the addon saved the cache file, records it as
|
|
63
|
+
* established, and (for auto-cache turns) renames the addon's pre-response
|
|
64
|
+
* file to the post-response path. Flips the turn's internal `committed`
|
|
65
|
+
* flag so the deferred `rollback` becomes a no-op.
|
|
110
66
|
*/
|
|
111
67
|
commitTurn(turn: TurnHandle, result: CommitResult): Promise<void>;
|
|
112
68
|
/**
|
|
113
69
|
* Roll back an in-flight turn — atomically deletes the on-disk cache
|
|
114
|
-
* file,
|
|
115
|
-
* `
|
|
116
|
-
* removes orphaned marker metadata.
|
|
117
|
-
* already been committed or rolled back is a
|
|
118
|
-
* calls. Handlers register this via
|
|
119
|
-
* runs regardless of how the handler exits
|
|
120
|
-
* itself via `commitTurn`).
|
|
70
|
+
* file, drops the addon's copy of the conversation through
|
|
71
|
+
* `options.discardCache`, clears the in-memory `initializedCaches` entry,
|
|
72
|
+
* releases the active-path ref, and removes orphaned marker metadata.
|
|
73
|
+
* Idempotent: a turn that has already been committed or rolled back is a
|
|
74
|
+
* no-op on subsequent calls. Handlers register this via
|
|
75
|
+
* `ctx.scope.defer(...)` so it runs regardless of how the handler exits
|
|
76
|
+
* (success branch removes itself via `commitTurn`).
|
|
121
77
|
*/
|
|
122
78
|
rollback(turn: TurnHandle): Promise<void>;
|
|
123
79
|
/**
|
|
124
80
|
* Non-destructive counterpart of `rollback`: releases locks and refs but
|
|
125
|
-
* keeps the committed disk cache
|
|
126
|
-
*
|
|
81
|
+
* keeps the committed disk cache for a retry. A cache this same turn
|
|
82
|
+
* created rolls back instead.
|
|
127
83
|
*/
|
|
128
84
|
releaseTurn(turn: TurnHandle): Promise<void>;
|
|
129
|
-
/**
|
|
130
|
-
* Forget the saved-message count for the turn's path, in memory and in the
|
|
131
|
-
* sidecar, without unlinking the cache file or clearing the init flag. Used
|
|
132
|
-
* when `decideCachedHistorySlice` detects a stale boundary
|
|
133
|
-
* (`clearStaleCount: true`) — the next turn re-sends the full history
|
|
134
|
-
* but the cache itself is still usable.
|
|
135
|
-
*/
|
|
136
|
-
dropStaleSavedCount(turn: TurnHandle): Promise<void>;
|
|
137
85
|
}
|
|
138
86
|
/**
|
|
139
87
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
140
88
|
* scope. `options.logger` is the per-instance logger the session emits
|
|
141
89
|
* through (typically `withRequestContext(getEngineLogger(), ctx)`);
|
|
142
|
-
* falls back to the module-scoped logger when omitted.
|
|
90
|
+
* falls back to the module-scoped logger when omitted. `options.discardCache`
|
|
91
|
+
* drops the addon's copy of a conversation whose cache the session rolls
|
|
92
|
+
* back; without it the addon writes that conversation to the deleted path at
|
|
93
|
+
* its next key switch or unload.
|
|
143
94
|
*/
|
|
144
95
|
export declare function createKvCacheSession(modelId: string, options?: {
|
|
145
96
|
logger?: Logger;
|
|
97
|
+
discardCache?: (cachePath: string) => Promise<void>;
|
|
146
98
|
}): KvCacheSession;
|
|
147
99
|
/**
|
|
148
100
|
* Atomically delete every layer of KV-cache state for a
|
|
@@ -161,10 +113,8 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
161
113
|
* Layers cleared, in order:
|
|
162
114
|
* 1. On-disk: `deleteCache(...)` removes the matching directory
|
|
163
115
|
* tree (or wipes and recreates the root for `all: true`).
|
|
164
|
-
* 2. `
|
|
165
|
-
*
|
|
166
|
-
* 3. `initializedCaches`: prefix-cleanup by the same removed
|
|
167
|
-
* directory, since it is keyed by the resolved cache path.
|
|
116
|
+
* 2. `initializedCaches`: prefix-cleanup by the removed directory,
|
|
117
|
+
* since it is keyed by the resolved cache path.
|
|
168
118
|
*
|
|
169
119
|
* Concurrency with in-flight turns: this delete does not take the
|
|
170
120
|
* per-cache-path write locks, so it races any turn holding a `TurnHandle`
|
|
@@ -173,9 +123,8 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
173
123
|
* `Map.delete` / `Set.delete` no-op on absent keys), but the layers are
|
|
174
124
|
* cleared without a lock, so an interleaving that lands the delete's
|
|
175
125
|
* in-memory cleanup after a concurrent turn has already renamed/committed
|
|
176
|
-
* its file splits state: the file stays on disk while its
|
|
177
|
-
*
|
|
178
|
-
* `savedCount=0`. Callers must not delete a key that is in active use.
|
|
126
|
+
* its file splits state: the file stays on disk while its init flag is
|
|
127
|
+
* cleared. Callers must not delete a key that is in active use.
|
|
179
128
|
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
180
129
|
* and clears each one under the cache-state lock.
|
|
181
130
|
*/
|
|
@@ -197,11 +146,6 @@ export declare function deleteKvCacheState(target: {
|
|
|
197
146
|
* @internal
|
|
198
147
|
*/
|
|
199
148
|
export declare const __kvCacheSessionTestHooks: {
|
|
200
|
-
getSavedCount(cachePath: string): number | undefined;
|
|
201
|
-
setSavedCountForTest(cachePath: string, count: number): void;
|
|
202
|
-
getToolBlockCachedForTest(cachePath: string): boolean;
|
|
203
|
-
getPrefixSidecarPathForTest(cachePath: string): string;
|
|
204
|
-
readPrefixSidecarForTest(cachePath: string): Promise<CachedPrefix | null>;
|
|
205
149
|
hasInitializedPath(cachePath: string): boolean;
|
|
206
150
|
getActivePathCountForTest(cachePath: string): number;
|
|
207
151
|
getLastAutoCacheSweepMsForTest(): number;
|