@qvac/inference 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +59 -40
- package/README.md +1 -1
- package/dist/api/audio-gen.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +3 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/index.d.ts +39 -38
- package/dist/api/index.js +38 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +6 -0
- package/dist/errors/index.js +13 -1
- package/dist/handlers/assess-model-fit.d.ts +1 -1
- package/dist/handlers/assess-model-fit.js +4 -4
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +2 -3
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.js +4 -2
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +1 -1
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +4 -4
- package/dist/{resources/model-fit → model-fit}/assess.js +3 -3
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +4 -4
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +1 -1
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +5 -5
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +2 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +6 -6
- package/dist/{resources/model-fit → model-fit}/floor.d.ts +2 -2
- package/dist/{resources/model-fit → model-fit}/floor.js +1 -1
- package/dist/{resources/model-fit → model-fit}/kv-cache.d.ts +1 -1
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +8 -10
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +14 -40
- package/dist/{resources/model-fit/types.d.ts → model-fit/platform.d.ts} +8 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +3 -3
- package/dist/{resources/model-fit/native-probe/create-fit-request.js → model-fit/request/create.js} +8 -8
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +2 -2
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +9 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.js +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -2
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.js +13 -10
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/nmt.d.ts +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/nmt.js +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/ocr.d.ts +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/ocr.js +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +2 -2
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +1 -1
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +3 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +1 -1
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +510 -0
- package/dist/models/registry/models.js +2946 -2410
- package/dist/models/registry/resource-profiles.js +203 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +60 -54
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +4 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +26 -25
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +4 -4
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +11 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +4 -4
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +12 -12
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +15 -14
- package/dist/plugins/builtin/index.js +15 -14
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +36 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +169 -78
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -17
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +39 -33
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +7 -6
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +16 -6
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +6 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +16 -16
- package/dist/plugins/builtin/moss-transcribe/plugin.js +4 -4
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +18 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +29 -29
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +4 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +19 -8
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +90 -34
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +45 -45
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +2 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +2 -2
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +47 -32
- package/dist/plugins/builtin/tts-ggml/plugin.js +3 -3
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +34 -33
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +4 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +4 -3
- package/dist/plugins/ops/kv-cache-utils.js +15 -8
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +14 -5
- package/dist/plugins/ops/transcribe.js +1 -1
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +2 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +3 -3
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +3 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +14 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +6 -0
- package/dist/schemas/assess-model-fit.js +4 -4
- package/dist/schemas/audio-gen.d.ts +13 -1
- package/dist/schemas/audio-gen.js +3 -3
- package/dist/schemas/batch-completion-stream.d.ts +8 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +1 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +478 -100
- package/dist/schemas/common.js +35 -32
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +11 -2
- package/dist/schemas/completion-stream.js +37 -10
- package/dist/schemas/config.d.ts +849 -6
- package/dist/schemas/config.js +12 -10
- package/dist/schemas/decision.d.ts +8 -0
- package/dist/schemas/decision.js +14 -0
- package/dist/schemas/download-asset.d.ts +6 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +2 -1
- package/dist/schemas/engine-addon-map.js +4 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +6 -1
- package/dist/schemas/index.d.ts +51 -50
- package/dist/schemas/index.js +50 -49
- package/dist/schemas/llamacpp-config.d.ts +182 -6
- package/dist/schemas/llamacpp-config.js +102 -7
- package/dist/schemas/load-model.d.ts +3176 -646
- package/dist/schemas/load-model.js +34 -1
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +195 -1
- package/dist/schemas/model-config-schemas.js +14 -12
- package/dist/schemas/model-fit-policy.d.ts +1 -1
- package/dist/schemas/model-info.js +2 -2
- package/dist/schemas/model-resource-profile.d.ts +1 -0
- package/dist/schemas/model-resource-profile.js +1 -1
- package/dist/schemas/model-src-utils.d.ts +6 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +5 -1
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +2 -0
- package/dist/schemas/ocr.js +1 -1
- package/dist/schemas/plugin.d.ts +9 -3
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +8 -0
- package/dist/schemas/registry.js +2 -1
- package/dist/schemas/reload-config.d.ts +6 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/sdcpp-config.d.ts +56 -0
- package/dist/schemas/sdcpp-config.js +33 -15
- package/dist/schemas/text-to-speech.d.ts +61 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +16 -1
- package/dist/schemas/transcription.d.ts +1 -1
- package/dist/schemas/transcription.js +1 -1
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +20 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/surface.d.ts +2 -1
- package/dist/surface.js +2 -1
- package/dist/utils/addon-responses.d.ts +2 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/tts-stats.d.ts +1 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +53 -45
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit → model-fit}/kv-cache.js +0 -0
- /package/dist/{resources/model-fit → model-fit}/platform.js +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
|
@@ -53,7 +53,7 @@ export declare const whisperPlugin: {
|
|
|
53
53
|
max_speech_duration_s: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
54
54
|
speech_pad_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
55
55
|
samples_overlap: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
56
|
-
}, import("zod/v4/core
|
|
56
|
+
}, import("zod/v4/core").$strip>>;
|
|
57
57
|
audio_format: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
58
58
|
f32le: "f32le";
|
|
59
59
|
s16le: "s16le";
|
|
@@ -63,11 +63,11 @@ export declare const whisperPlugin: {
|
|
|
63
63
|
use_gpu: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
64
64
|
flash_attn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
65
65
|
gpu_device: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
66
|
-
}, import("zod/v4/core
|
|
66
|
+
}, import("zod/v4/core").$strip>>;
|
|
67
67
|
miscConfig: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
68
68
|
caption_enabled: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
69
69
|
seed: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
70
|
-
}, import("zod/v4/core
|
|
70
|
+
}, import("zod/v4/core").$strip>>;
|
|
71
71
|
vadModelSrc: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
|
|
72
72
|
src: import("zod").ZodString;
|
|
73
73
|
name: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -91,6 +91,7 @@ export declare const whisperPlugin: {
|
|
|
91
91
|
"bci-whispercpp-transcription": "bci-whispercpp-transcription";
|
|
92
92
|
"llamacpp-embedding": "llamacpp-embedding";
|
|
93
93
|
"llamacpp-decision": "llamacpp-decision";
|
|
94
|
+
"llamacpp-rerank": "llamacpp-rerank";
|
|
94
95
|
"nmtcpp-translation": "nmtcpp-translation";
|
|
95
96
|
"onnx-tts": "onnx-tts";
|
|
96
97
|
"tts-ggml": "tts-ggml";
|
|
@@ -108,8 +109,8 @@ export declare const whisperPlugin: {
|
|
|
108
109
|
parakeet: "parakeet";
|
|
109
110
|
classification: "classification";
|
|
110
111
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
111
|
-
}, import("zod/v4/core
|
|
112
|
-
}, import("zod/v4/core
|
|
112
|
+
}, import("zod/v4/core").$strip>]>>;
|
|
113
|
+
}, import("zod/v4/core").$strip>;
|
|
113
114
|
assessFit: typeof import("node_modules/@qvac/asr-ggml/lib/fit").assessFit;
|
|
114
115
|
resolveConfig(cfg: WhisperConfig, ctx: ResolveContext): Promise<{
|
|
115
116
|
config: {
|
|
@@ -247,15 +248,15 @@ export declare const whisperPlugin: {
|
|
|
247
248
|
audioChunk: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
248
249
|
type: import("zod").ZodLiteral<"base64">;
|
|
249
250
|
value: import("zod").ZodString;
|
|
250
|
-
}, import("zod/v4/core
|
|
251
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
251
252
|
type: import("zod").ZodLiteral<"filePath">;
|
|
252
253
|
value: import("zod").ZodString;
|
|
253
|
-
}, import("zod/v4/core
|
|
254
|
+
}, import("zod/v4/core").$strip>], "type">;
|
|
254
255
|
hotwords: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString>>;
|
|
255
256
|
maxNewTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
256
257
|
type: import("zod").ZodLiteral<"transcribe">;
|
|
257
258
|
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
258
|
-
}, import("zod/v4/core
|
|
259
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
259
260
|
text: import("zod").ZodOptional<import("zod").ZodString>;
|
|
260
261
|
done: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
261
262
|
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -286,7 +287,7 @@ export declare const whisperPlugin: {
|
|
|
286
287
|
totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
287
288
|
encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
288
289
|
encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
289
|
-
}, import("zod/v4/core
|
|
290
|
+
}, import("zod/v4/core").$strip>>;
|
|
290
291
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
291
292
|
segment: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
292
293
|
text: import("zod").ZodString;
|
|
@@ -300,10 +301,10 @@ export declare const whisperPlugin: {
|
|
|
300
301
|
speakerId: import("zod").ZodNumber;
|
|
301
302
|
startMs: import("zod").ZodNumber;
|
|
302
303
|
endMs: import("zod").ZodNumber;
|
|
303
|
-
}, import("zod/v4/core
|
|
304
|
+
}, import("zod/v4/core").$strip>>>;
|
|
304
305
|
isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
305
306
|
startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
306
|
-
}, import("zod/v4/core
|
|
307
|
+
}, import("zod/v4/core").$strip>>;
|
|
307
308
|
vad: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
308
309
|
speakerId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
309
310
|
speaking: import("zod").ZodBoolean;
|
|
@@ -313,13 +314,13 @@ export declare const whisperPlugin: {
|
|
|
313
314
|
energy: "energy";
|
|
314
315
|
sortformer: "sortformer";
|
|
315
316
|
}>>;
|
|
316
|
-
}, import("zod/v4/core
|
|
317
|
+
}, import("zod/v4/core").$strip>>;
|
|
317
318
|
endOfTurn: import("zod").ZodOptional<import("zod").ZodPreprocess<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
318
319
|
source: import("zod").ZodLiteral<"whisper">;
|
|
319
320
|
silenceDurationMs: import("zod").ZodNumber;
|
|
320
|
-
}, import("zod/v4/core
|
|
321
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
321
322
|
source: import("zod").ZodLiteral<"parakeet">;
|
|
322
|
-
}, import("zod/v4/core
|
|
323
|
+
}, import("zod/v4/core").$strip>], "source">, unknown>>;
|
|
323
324
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
324
325
|
selectedBackend: import("zod").ZodString;
|
|
325
326
|
selectedDevice: import("zod").ZodEnum<{
|
|
@@ -341,7 +342,7 @@ export declare const whisperPlugin: {
|
|
|
341
342
|
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
342
343
|
name: import("zod").ZodString;
|
|
343
344
|
version: import("zod").ZodOptional<import("zod").ZodString>;
|
|
344
|
-
}, import("zod/v4/core
|
|
345
|
+
}, import("zod/v4/core").$strip>>;
|
|
345
346
|
gpuId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
346
347
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
347
348
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -350,7 +351,7 @@ export declare const whisperPlugin: {
|
|
|
350
351
|
gpu: "gpu";
|
|
351
352
|
}>>;
|
|
352
353
|
reason: import("zod").ZodString;
|
|
353
|
-
}, import("zod/v4/core
|
|
354
|
+
}, import("zod/v4/core").$strip>>;
|
|
354
355
|
probe: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
355
356
|
status: import("zod").ZodEnum<{
|
|
356
357
|
unknown: "unknown";
|
|
@@ -359,10 +360,10 @@ export declare const whisperPlugin: {
|
|
|
359
360
|
}>;
|
|
360
361
|
backend: import("zod").ZodString;
|
|
361
362
|
reason: import("zod").ZodOptional<import("zod").ZodString>;
|
|
362
|
-
}, import("zod/v4/core
|
|
363
|
-
}, import("zod/v4/core
|
|
363
|
+
}, import("zod/v4/core").$strip>>;
|
|
364
|
+
}, import("zod/v4/core").$strip>>;
|
|
364
365
|
type: import("zod").ZodLiteral<"transcribe">;
|
|
365
|
-
}, import("zod/v4/core
|
|
366
|
+
}, import("zod/v4/core").$strip>>;
|
|
366
367
|
transcribeStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
367
368
|
modelId: import("zod").ZodString;
|
|
368
369
|
prompt: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -387,9 +388,9 @@ export declare const whisperPlugin: {
|
|
|
387
388
|
chunkLeftContextMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
388
389
|
chunkRightContextMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
389
390
|
spkCacheUpdatePeriod: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
390
|
-
}, import("zod/v4/core
|
|
391
|
+
}, import("zod/v4/core").$strip>>;
|
|
391
392
|
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
392
|
-
}, import("zod/v4/core
|
|
393
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
393
394
|
text: import("zod").ZodOptional<import("zod").ZodString>;
|
|
394
395
|
done: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
395
396
|
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -420,7 +421,7 @@ export declare const whisperPlugin: {
|
|
|
420
421
|
totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
421
422
|
encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
422
423
|
encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
423
|
-
}, import("zod/v4/core
|
|
424
|
+
}, import("zod/v4/core").$strip>>;
|
|
424
425
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
425
426
|
segment: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
426
427
|
text: import("zod").ZodString;
|
|
@@ -434,10 +435,10 @@ export declare const whisperPlugin: {
|
|
|
434
435
|
speakerId: import("zod").ZodNumber;
|
|
435
436
|
startMs: import("zod").ZodNumber;
|
|
436
437
|
endMs: import("zod").ZodNumber;
|
|
437
|
-
}, import("zod/v4/core
|
|
438
|
+
}, import("zod/v4/core").$strip>>>;
|
|
438
439
|
isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
439
440
|
startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
440
|
-
}, import("zod/v4/core
|
|
441
|
+
}, import("zod/v4/core").$strip>>;
|
|
441
442
|
vad: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
442
443
|
speakerId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
443
444
|
speaking: import("zod").ZodBoolean;
|
|
@@ -447,13 +448,13 @@ export declare const whisperPlugin: {
|
|
|
447
448
|
energy: "energy";
|
|
448
449
|
sortformer: "sortformer";
|
|
449
450
|
}>>;
|
|
450
|
-
}, import("zod/v4/core
|
|
451
|
+
}, import("zod/v4/core").$strip>>;
|
|
451
452
|
endOfTurn: import("zod").ZodOptional<import("zod").ZodPreprocess<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
452
453
|
source: import("zod").ZodLiteral<"whisper">;
|
|
453
454
|
silenceDurationMs: import("zod").ZodNumber;
|
|
454
|
-
}, import("zod/v4/core
|
|
455
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
455
456
|
source: import("zod").ZodLiteral<"parakeet">;
|
|
456
|
-
}, import("zod/v4/core
|
|
457
|
+
}, import("zod/v4/core").$strip>], "source">, unknown>>;
|
|
457
458
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
458
459
|
selectedBackend: import("zod").ZodString;
|
|
459
460
|
selectedDevice: import("zod").ZodEnum<{
|
|
@@ -475,7 +476,7 @@ export declare const whisperPlugin: {
|
|
|
475
476
|
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
476
477
|
name: import("zod").ZodString;
|
|
477
478
|
version: import("zod").ZodOptional<import("zod").ZodString>;
|
|
478
|
-
}, import("zod/v4/core
|
|
479
|
+
}, import("zod/v4/core").$strip>>;
|
|
479
480
|
gpuId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
480
481
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
481
482
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -484,7 +485,7 @@ export declare const whisperPlugin: {
|
|
|
484
485
|
gpu: "gpu";
|
|
485
486
|
}>>;
|
|
486
487
|
reason: import("zod").ZodString;
|
|
487
|
-
}, import("zod/v4/core
|
|
488
|
+
}, import("zod/v4/core").$strip>>;
|
|
488
489
|
probe: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
489
490
|
status: import("zod").ZodEnum<{
|
|
490
491
|
unknown: "unknown";
|
|
@@ -493,13 +494,13 @@ export declare const whisperPlugin: {
|
|
|
493
494
|
}>;
|
|
494
495
|
backend: import("zod").ZodString;
|
|
495
496
|
reason: import("zod").ZodOptional<import("zod").ZodString>;
|
|
496
|
-
}, import("zod/v4/core
|
|
497
|
-
}, import("zod/v4/core
|
|
497
|
+
}, import("zod/v4/core").$strip>>;
|
|
498
|
+
}, import("zod/v4/core").$strip>>;
|
|
498
499
|
type: import("zod").ZodLiteral<"transcribeStream">;
|
|
499
|
-
}, import("zod/v4/core
|
|
500
|
+
}, import("zod/v4/core").$strip>>;
|
|
500
501
|
};
|
|
501
502
|
logging: {
|
|
502
|
-
module: () => Promise<typeof import("@qvac/asr-ggml/addonLogging
|
|
503
|
+
module: () => Promise<typeof import("@qvac/asr-ggml/addonLogging")>;
|
|
503
504
|
namespace: "@qvac/asr-ggml";
|
|
504
505
|
};
|
|
505
506
|
};
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import ASRGgml from '@qvac/asr-ggml';
|
|
2
2
|
import { definePlugin, defineHandler, defineDuplexHandler, transcribeRequestSchema, transcribeResponseSchema, transcribeStreamRequestSchema, transcribeStreamResponseSchema, ModelType, whisperConfigSchema, ADDON_ASR } from '../../../schemas/index.js';
|
|
3
|
-
import { transcribe, transcribeStream } from '
|
|
3
|
+
import { transcribe, transcribeStream } from '../../../plugins/ops/transcribe.js';
|
|
4
4
|
import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
|
|
5
5
|
import { attachBackendDiagnostics } from '../../../profiling/backend-diagnostics.js';
|
|
6
|
-
import { buildWhisperEngineConfig } from '
|
|
7
|
-
import { createAsrModelLogger } from '
|
|
6
|
+
import { buildWhisperEngineConfig } from '../../../plugins/builtin/asr-ggml/config.js';
|
|
7
|
+
import { createAsrModelLogger } from '../../../plugins/builtin/asr-ggml/logging.js';
|
|
8
8
|
function createWhisperModel(modelId, modelPath, whisperConfig, vadModelPath) {
|
|
9
9
|
const logger = createAsrModelLogger(modelId, ModelType.whispercppTranscription);
|
|
10
10
|
const model = new ASRGgml({
|
|
@@ -186,7 +186,7 @@ export const whisperPlugin = definePlugin({
|
|
|
186
186
|
})
|
|
187
187
|
},
|
|
188
188
|
logging: {
|
|
189
|
-
module: () => import('@qvac/asr-ggml/addonLogging
|
|
189
|
+
module: () => import('@qvac/asr-ggml/addonLogging'),
|
|
190
190
|
namespace: ADDON_ASR
|
|
191
191
|
}
|
|
192
192
|
});
|
package/dist/plugins/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as hostApi from '../api/index.js';
|
|
2
2
|
import type { QvacPlugin } from '../schemas/plugin.js';
|
|
3
|
-
export { registerPlugin, registerPlugins, ensureAddonLoggerReady, getPlugin, getPluginHandler, getTurboVecIndexProvider, hasPlugin, getAllPlugins, clearPlugins, unregisterPlugin } from '
|
|
3
|
+
export { registerPlugin, registerPlugins, ensureAddonLoggerReady, getPlugin, getPluginHandler, getTurboVecIndexProvider, hasPlugin, getAllPlugins, clearPlugins, unregisterPlugin } from '../plugins/registry.js';
|
|
4
4
|
/**
|
|
5
5
|
* Register a set of plugins and return the host API bound to them. The
|
|
6
6
|
* ergonomic entry point for assembling an explicit plugin subset — we ship
|
package/dist/plugins/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { registerPlugins } from '
|
|
1
|
+
import { registerPlugins } from '../plugins/registry.js';
|
|
2
2
|
import * as hostApi from '../api/index.js';
|
|
3
|
-
export { registerPlugin, registerPlugins, ensureAddonLoggerReady, getPlugin, getPluginHandler, getTurboVecIndexProvider, hasPlugin, getAllPlugins, clearPlugins, unregisterPlugin } from '
|
|
3
|
+
export { registerPlugin, registerPlugins, ensureAddonLoggerReady, getPlugin, getPluginHandler, getTurboVecIndexProvider, hasPlugin, getAllPlugins, clearPlugins, unregisterPlugin } from '../plugins/registry.js';
|
|
4
4
|
/**
|
|
5
5
|
* Register a set of plugins and return the host API bound to them. The
|
|
6
6
|
* ergonomic entry point for assembling an explicit plugin subset — we ship
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { type CacheMessage } from '../../utils/index.js';
|
|
2
|
+
import type { Tool } from '../../schemas/tools.js';
|
|
2
3
|
export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
|
|
3
4
|
export declare function generateConfigHash(systemPrompt: string | null): string;
|
|
4
|
-
export declare function generateCacheKey(messages: CacheMessage[]): string;
|
|
5
|
+
export declare function generateCacheKey(messages: CacheMessage[], tools?: readonly Tool[]): string;
|
|
5
6
|
export declare function getCacheFilePath(modelId: string, configHash: string, cacheKey: string): Promise<string>;
|
|
6
|
-
export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
|
|
7
|
+
export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[], tools?: readonly Tool[]): Promise<{
|
|
7
8
|
cacheKey: string;
|
|
8
9
|
cachePath: string;
|
|
9
10
|
} | null>;
|
|
10
|
-
export declare function getCurrentCacheInfo(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
|
|
11
|
+
export declare function getCurrentCacheInfo(modelId: string, configHash: string, currentHistory: CacheMessage[], tools?: readonly Tool[]): Promise<{
|
|
11
12
|
cacheKey: string;
|
|
12
13
|
cachePath: string;
|
|
13
14
|
}>;
|
|
@@ -5,7 +5,7 @@ import { getAutoCacheLookupHistory, getKVCacheDir, validateAndJoinPath } from '.
|
|
|
5
5
|
import { getEngineLogger } from '../../logging/index.js';
|
|
6
6
|
import { PathTraversalError } from '../../errors/index.js';
|
|
7
7
|
import { Buffer } from 'bare-buffer';
|
|
8
|
-
import { markAutoCacheKey } from '
|
|
8
|
+
import { markAutoCacheKey } from '../../plugins/ops/kv-cache-retention.js';
|
|
9
9
|
const logger = getEngineLogger();
|
|
10
10
|
// In-memory KV-cache state lives in `KvCacheSession`, its single mutation
|
|
11
11
|
// point. This module keeps only the pure path / hash utilities that don't
|
|
@@ -24,9 +24,14 @@ export function generateConfigHash(systemPrompt) {
|
|
|
24
24
|
hash.update(Buffer.from(JSON.stringify({ systemPrompt, tools: [] }), 'utf8'));
|
|
25
25
|
return hash.digest('hex').substring(0, 16);
|
|
26
26
|
}
|
|
27
|
-
export function generateCacheKey(messages) {
|
|
27
|
+
export function generateCacheKey(messages, tools) {
|
|
28
28
|
const hash = crypto.createHash('sha-256');
|
|
29
|
-
|
|
29
|
+
// Before the first assistant reply, the rendered tools are part of the
|
|
30
|
+
// reusable prefix. Keep those primes isolated; established conversations
|
|
31
|
+
// retain history-only keys so the addon can trim a changed tool block.
|
|
32
|
+
const historyString = JSON.stringify(tools?.length && !messages.some((message) => message.role === 'assistant')
|
|
33
|
+
? { messages, tools }
|
|
34
|
+
: messages);
|
|
30
35
|
const historyBuffer = Buffer.from(historyString, 'utf8');
|
|
31
36
|
hash.update(historyBuffer);
|
|
32
37
|
const hashString = hash.digest('hex');
|
|
@@ -50,12 +55,14 @@ export async function getCacheFilePath(modelId, configHash, cacheKey) {
|
|
|
50
55
|
return cachePath;
|
|
51
56
|
}
|
|
52
57
|
// Used for auto-generated cache key
|
|
53
|
-
export async function findMatchingCache(modelId, configHash, currentHistory) {
|
|
54
|
-
if (currentHistory.length
|
|
58
|
+
export async function findMatchingCache(modelId, configHash, currentHistory, tools) {
|
|
59
|
+
if (currentHistory.length === 0) {
|
|
55
60
|
return null;
|
|
56
61
|
}
|
|
57
62
|
const previousHistory = getAutoCacheLookupHistory(currentHistory);
|
|
58
|
-
|
|
63
|
+
if (previousHistory.length === 0 && !tools?.length)
|
|
64
|
+
return null;
|
|
65
|
+
const cacheKey = generateCacheKey(previousHistory, tools);
|
|
59
66
|
const cachePath = resolveCacheFilePath(modelId, configHash, cacheKey);
|
|
60
67
|
try {
|
|
61
68
|
await fsPromises.access(cachePath);
|
|
@@ -66,8 +73,8 @@ export async function findMatchingCache(modelId, configHash, currentHistory) {
|
|
|
66
73
|
return null;
|
|
67
74
|
}
|
|
68
75
|
}
|
|
69
|
-
export async function getCurrentCacheInfo(modelId, configHash, currentHistory) {
|
|
70
|
-
const cacheKey = generateCacheKey(currentHistory);
|
|
76
|
+
export async function getCurrentCacheInfo(modelId, configHash, currentHistory, tools) {
|
|
77
|
+
const cacheKey = generateCacheKey(currentHistory, tools);
|
|
71
78
|
await markAutoCacheKey(cacheKey);
|
|
72
79
|
const cachePath = await getCacheFilePath(modelId, configHash, cacheKey);
|
|
73
80
|
return { cacheKey, cachePath };
|
|
@@ -1,8 +1,14 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
1
2
|
import { type LoadModelServerParams } from '../../schemas/index.js';
|
|
2
3
|
import type { LoadTimingStats } from '../../profiling/types.js';
|
|
3
4
|
export interface LoadModelResult {
|
|
4
5
|
timing?: LoadTimingStats;
|
|
5
6
|
}
|
|
6
|
-
export
|
|
7
|
+
export interface LoadModelOptions {
|
|
7
8
|
collectTiming?: boolean;
|
|
8
|
-
|
|
9
|
+
request?: {
|
|
10
|
+
signal: AbortSignal;
|
|
11
|
+
requestId: string;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export declare function loadModel(params: LoadModelServerParams, options?: LoadModelOptions): Promise<LoadModelResult>;
|
|
@@ -4,11 +4,12 @@ import { forwardBackendDiagnostics } from '../../profiling/backend-diagnostics.j
|
|
|
4
4
|
import { isModelLoaded, registerModel } from '../../runtime/model-registry.js';
|
|
5
5
|
import { startLogBuffering, stopLogBufferingWithTimeout } from '../../runtime/logging-stream-registry.js';
|
|
6
6
|
import { detectShardedModel, generateShardFilenames, validateShardedModelCache } from '../../utils/index.js';
|
|
7
|
-
import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError } from '../../errors/index.js';
|
|
8
|
-
import { ensureAddonLoggerReady, getPlugin } from '
|
|
9
|
-
import { runAdvisoryFitCheck } from '../../
|
|
7
|
+
import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError, InferenceCancelledError } from '../../errors/index.js';
|
|
8
|
+
import { ensureAddonLoggerReady, getPlugin } from '../../plugins/index.js';
|
|
9
|
+
import { runAdvisoryFitCheck } from '../../model-fit/load-check.js';
|
|
10
10
|
import { refusesLoad, resolveModelFitPolicy, runsProbe } from '../../schemas/model-fit-policy.js';
|
|
11
11
|
import { getConfig } from '../../runtime/state.js';
|
|
12
|
+
import { resolveToolDialect } from '../../utils/tools/dialect.js';
|
|
12
13
|
import { promises as fsPromises } from 'bare-fs';
|
|
13
14
|
import path from 'bare-path';
|
|
14
15
|
import { getEngineLogger } from '../../logging/index.js';
|
|
@@ -59,6 +60,7 @@ export async function loadModel(params, options) {
|
|
|
59
60
|
// the real load uses, and before `createModel()` so it never competes with
|
|
60
61
|
// the native load for device memory.
|
|
61
62
|
const fitPolicy = resolveModelFitPolicy(modelFitPolicy, getConfig().modelFitPolicy);
|
|
63
|
+
const request = options?.request;
|
|
62
64
|
const fitProbe = runsProbe(fitPolicy)
|
|
63
65
|
? await runAdvisoryFitCheck({
|
|
64
66
|
modelId,
|
|
@@ -67,8 +69,11 @@ export async function loadModel(params, options) {
|
|
|
67
69
|
modelConfig,
|
|
68
70
|
artifacts,
|
|
69
71
|
isShardedModel
|
|
70
|
-
})
|
|
72
|
+
}, request !== undefined ? { signal: request.signal } : {})
|
|
71
73
|
: undefined;
|
|
74
|
+
if (request?.signal.aborted === true) {
|
|
75
|
+
throw new InferenceCancelledError(request.requestId);
|
|
76
|
+
}
|
|
72
77
|
if (refusesLoad(fitPolicy, fitProbe?.verdict)) {
|
|
73
78
|
throw new ModelFitRefusedError(modelType, fitProbe.reason, fitProbe.message);
|
|
74
79
|
}
|
|
@@ -90,13 +95,17 @@ export async function loadModel(params, options) {
|
|
|
90
95
|
await result.model.load(false);
|
|
91
96
|
const modelInitializationTimeMs = options?.collectTiming ? nowMs() - initStart : undefined;
|
|
92
97
|
logger.info(`${modelType} model ${modelId} loaded`);
|
|
98
|
+
const toolDialect = 'completionStream' in plugin.handlers
|
|
99
|
+
? await resolveToolDialect(modelName, modelPath)
|
|
100
|
+
: undefined;
|
|
93
101
|
registerModel(modelId, {
|
|
94
102
|
model: result.model,
|
|
95
103
|
path: modelPath,
|
|
96
104
|
config: modelConfig,
|
|
97
105
|
modelType: modelType,
|
|
98
106
|
name: modelName,
|
|
99
|
-
fitProbe
|
|
107
|
+
fitProbe,
|
|
108
|
+
toolDialect
|
|
100
109
|
});
|
|
101
110
|
const loadResult = modelInitializationTimeMs !== undefined ? { timing: { modelInitializationTimeMs } } : {};
|
|
102
111
|
return forwardBackendDiagnostics(loadResult, result);
|
|
@@ -62,7 +62,7 @@ import { assertMetadataSupported, toTranscribeSegment } from '../../utils/transc
|
|
|
62
62
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
63
63
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
64
64
|
import { isEndOfTurnEvent, isVadEvent, toEndOfTurnEvent, toVadStateEvent } from '../../utils/asr-events.js';
|
|
65
|
-
import { buildWhisperReloadConfig } from '
|
|
65
|
+
import { buildWhisperReloadConfig } from '../../plugins/builtin/asr-ggml/config.js';
|
|
66
66
|
export { assertMetadataSupported, toTranscribeSegment };
|
|
67
67
|
function hasRunStreaming(model) {
|
|
68
68
|
return 'runStreaming' in model && typeof model.runStreaming === 'function';
|
|
@@ -55,14 +55,14 @@ import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LAN
|
|
|
55
55
|
import { getLangName, detectOne } from '@qvac/langdetect-text';
|
|
56
56
|
import { nowMs } from '../../profiling/index.js';
|
|
57
57
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
58
|
-
import { buildNmtTranslationStats, markNmtCountersStale } from '
|
|
58
|
+
import { buildNmtTranslationStats, markNmtCountersStale } from '../../plugins/ops/translate-stats.js';
|
|
59
59
|
import { ModelNotFoundError, ModelTypeMismatchError, TranslationFailedError } from '../../errors/index.js';
|
|
60
60
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
61
|
-
import { isAddonContextOverflowError } from '
|
|
61
|
+
import { isAddonContextOverflowError } from '../../plugins/builtin/llamacpp-completion/ops/context-overflow.js';
|
|
62
62
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
63
63
|
import { getModelParallel } from '../../utils/config-transform.js';
|
|
64
64
|
import { getEngineLogger } from '../../logging/index.js';
|
|
65
|
-
import { isTranslatePsyModel, buildTranslatePsyMessages, TRANSLATEPSY_GENERATION_PARAMS } from '
|
|
65
|
+
import { isTranslatePsyModel, buildTranslatePsyMessages, TRANSLATEPSY_GENERATION_PARAMS } from '../../plugins/ops/translatepsy-utils.js';
|
|
66
66
|
const AFRICAN_LANGUAGE_ALIASES = new Map([
|
|
67
67
|
['af', 'afr_Latn'],
|
|
68
68
|
['sw', 'swh_Latn'],
|
|
@@ -3,7 +3,7 @@ import path from 'bare-path';
|
|
|
3
3
|
import { unregisterModel } from '../../runtime/model-registry.js';
|
|
4
4
|
import { getRequestRegistry } from '../../runtime/index.js';
|
|
5
5
|
import { unregisterAllLoggingStreams } from '../../runtime/logging-stream-registry.js';
|
|
6
|
-
import { clearFinetuneRuntimeState } from '
|
|
6
|
+
import { clearFinetuneRuntimeState } from '../../plugins/builtin/llamacpp-completion/ops/finetune.js';
|
|
7
7
|
import { unregisterAddonLogger, getEngineLogger } from '../../logging/index.js';
|
|
8
8
|
import { unloadModelParamsSchema } from '../../schemas/index.js';
|
|
9
9
|
import { ModelNotLoadedError } from '../../errors/index.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getTurboVecIndexProvider } from '
|
|
1
|
+
import { getTurboVecIndexProvider } from '../plugins/registry.js';
|
|
2
2
|
import { VECTOR_ID_RESERVED, VectorIndexStorage } from '../schemas/index.js';
|
|
3
3
|
// The native index pads short result rows with UINT64_MAX.
|
|
4
4
|
const PADDING_ID = BigInt(VECTOR_ID_RESERVED);
|
|
@@ -88,11 +88,11 @@ export function getTurboVecBackend() {
|
|
|
88
88
|
if (!provider)
|
|
89
89
|
return undefined;
|
|
90
90
|
return {
|
|
91
|
-
create(dim, storage) {
|
|
92
|
-
return openIndex(provider.create({ dim, storage }), storage);
|
|
91
|
+
async create(dim, storage) {
|
|
92
|
+
return openIndex(await provider.create({ dim, storage }), storage);
|
|
93
93
|
},
|
|
94
|
-
load(path) {
|
|
95
|
-
const index = provider.load(path);
|
|
94
|
+
async load(path) {
|
|
95
|
+
const index = await provider.load(path);
|
|
96
96
|
return openIndex(index, storageFromBitWidth(index.bitWidth));
|
|
97
97
|
}
|
|
98
98
|
};
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Accumulates profiling statistics and maintains a bounded buffer of recent events.
|
|
3
3
|
* Uses a ring buffer for O(1) event insertion.
|
|
4
4
|
*/
|
|
5
|
-
import type { ProfilingEvent, AggregatedStats } from '
|
|
5
|
+
import type { ProfilingEvent, AggregatedStats } from '../profiling/types.js';
|
|
6
6
|
export declare function createAggregator(maxRecentEvents?: number): void;
|
|
7
7
|
export declare function recordEvent(event: ProfilingEvent, storeInBuffer?: boolean): void;
|
|
8
8
|
export declare function getAggregates(): Record<string, AggregatedStats>;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Uses a ring buffer for O(1) event insertion.
|
|
4
4
|
*/
|
|
5
5
|
import { getGlobalSingleton } from '../utils/global-singleton.js';
|
|
6
|
-
import { createRingBuffer, ringBufferPush, ringBufferToArray, ringBufferClear, ringBufferResize, ringBufferDroppedCount } from '
|
|
6
|
+
import { createRingBuffer, ringBufferPush, ringBufferToArray, ringBufferClear, ringBufferResize, ringBufferDroppedCount } from '../profiling/ring-buffer.js';
|
|
7
7
|
const DEFAULT_MAX_RECENT_EVENTS = 1000;
|
|
8
8
|
const AGGREGATOR_STATE_KEY = Symbol.for('@qvac/inference:profiler-aggregator-state');
|
|
9
9
|
function getAggregatorState() {
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* 3. Disabled default
|
|
8
8
|
*/
|
|
9
9
|
import type { ProfilerMode, PerCallProfiling } from '../schemas/index.js';
|
|
10
|
-
import type { ProfilerRuntimeOptions, ProfilingEvent, AggregatedStats } from '
|
|
10
|
+
import type { ProfilerRuntimeOptions, ProfilingEvent, AggregatedStats } from '../profiling/types.js';
|
|
11
11
|
export interface ResolvedProfilerConfig {
|
|
12
12
|
enabled: boolean;
|
|
13
13
|
mode: ProfilerMode;
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* 3. Disabled default
|
|
8
8
|
*/
|
|
9
9
|
import { getGlobalSingleton } from '../utils/global-singleton.js';
|
|
10
|
-
import { createAggregator, recordEvent as aggregatorRecord, getAggregates as aggregatorGetAggregates, getRecentEvents as aggregatorGetRecentEvents, clearAggregator, setMaxRecentEvents } from '
|
|
11
|
-
import { nowMs, generateProfileId } from '
|
|
10
|
+
import { createAggregator, recordEvent as aggregatorRecord, getAggregates as aggregatorGetAggregates, getRecentEvents as aggregatorGetRecentEvents, clearAggregator, setMaxRecentEvents } from '../profiling/aggregator.js';
|
|
11
|
+
import { nowMs, generateProfileId } from '../profiling/clock.js';
|
|
12
12
|
const DEFAULT_CONFIG = {
|
|
13
13
|
enabled: false,
|
|
14
14
|
mode: 'summary',
|
package/dist/profiling/events.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { nowMs } from '
|
|
2
|
-
import { record } from '
|
|
1
|
+
import { nowMs } from '../profiling/clock.js';
|
|
2
|
+
import { record } from '../profiling/controller.js';
|
|
3
3
|
export function recordPhase(base, phase, ms, extra) {
|
|
4
4
|
if (ms === undefined)
|
|
5
5
|
return;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* JSON, table, and summary export utilities for profiling data.
|
|
3
3
|
*/
|
|
4
|
-
import type { ProfilerExport } from '
|
|
4
|
+
import type { ProfilerExport } from '../profiling/types.js';
|
|
5
5
|
export declare function exportJSON(options?: {
|
|
6
6
|
includeRecentEvents?: boolean;
|
|
7
7
|
}): ProfilerExport;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* JSON, table, and summary export utilities for profiling data.
|
|
3
3
|
*/
|
|
4
|
-
import { getEffectiveConfig, getAggregates, getRecentEvents } from '
|
|
5
|
-
import { getDroppedCount, getEventCount } from '
|
|
6
|
-
import { nowMs } from '
|
|
4
|
+
import { getEffectiveConfig, getAggregates, getRecentEvents } from '../profiling/controller.js';
|
|
5
|
+
import { getDroppedCount, getEventCount } from '../profiling/aggregator.js';
|
|
6
|
+
import { nowMs } from '../profiling/clock.js';
|
|
7
7
|
export function exportJSON(options) {
|
|
8
8
|
const config = getEffectiveConfig();
|
|
9
9
|
const includeRecent = options?.includeRecentEvents ?? true;
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* profiler.disable();
|
|
12
12
|
* ```
|
|
13
13
|
*/
|
|
14
|
-
import * as controller from '
|
|
15
|
-
import type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats } from '
|
|
14
|
+
import * as controller from '../profiling/controller.js';
|
|
15
|
+
import type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats } from '../profiling/types.js';
|
|
16
16
|
export declare const profiler: {
|
|
17
17
|
/**
|
|
18
18
|
* Enables profiling and resets all previously aggregated data.
|
|
@@ -76,15 +76,15 @@ export declare const profiler: {
|
|
|
76
76
|
*/
|
|
77
77
|
clear: () => void;
|
|
78
78
|
};
|
|
79
|
-
export type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats, ProfilingEventKind } from '
|
|
79
|
+
export type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats, ProfilingEventKind } from '../profiling/types.js';
|
|
80
80
|
export type { ProfilerMode, ProfilerGPUResourceGauge, ProfilerResourceGauge } from '../schemas/index.js';
|
|
81
|
-
export { nowMs } from '
|
|
82
|
-
export { attachBackendDiagnostics } from '
|
|
83
|
-
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord, type ResolvedProfilerConfig } from '
|
|
84
|
-
export { recordEvent, clearAggregator } from '
|
|
85
|
-
export { exportJSON, exportTable, exportSummary } from '
|
|
86
|
-
export { createProfilingMeta, createProfilingDisabledMeta, injectProfilingMetaIntoObject, extractProfilingMeta, stripProfilingMeta } from '
|
|
87
|
-
export { recordPhase, recordFailure, recordServerBreakdownPhases, type BaseTimings, type BaseEvent } from '
|
|
88
|
-
export { createServerProfiler, type ServerProfiler } from '
|
|
89
|
-
export { profileReplyHandler, profileStreamHandler } from '
|
|
90
|
-
export { registerOperationMetrics, buildOperationEvent, type OperationMetricsConfig } from '
|
|
81
|
+
export { nowMs } from '../profiling/clock.js';
|
|
82
|
+
export { attachBackendDiagnostics } from '../profiling/backend-diagnostics.js';
|
|
83
|
+
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord, type ResolvedProfilerConfig } from '../profiling/controller.js';
|
|
84
|
+
export { recordEvent, clearAggregator } from '../profiling/aggregator.js';
|
|
85
|
+
export { exportJSON, exportTable, exportSummary } from '../profiling/exporters.js';
|
|
86
|
+
export { createProfilingMeta, createProfilingDisabledMeta, injectProfilingMetaIntoObject, extractProfilingMeta, stripProfilingMeta } from '../profiling/envelope.js';
|
|
87
|
+
export { recordPhase, recordFailure, recordServerBreakdownPhases, type BaseTimings, type BaseEvent } from '../profiling/events.js';
|
|
88
|
+
export { createServerProfiler, type ServerProfiler } from '../profiling/profiler.js';
|
|
89
|
+
export { profileReplyHandler, profileStreamHandler } from '../profiling/operation-wrappers.js';
|
|
90
|
+
export { registerOperationMetrics, buildOperationEvent, type OperationMetricsConfig } from '../profiling/operation-metrics.js';
|