@qvac/inference 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +89 -41
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +5 -2
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -2
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +39 -37
- package/dist/api/index.js +38 -36
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +6 -5
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +9 -0
- package/dist/errors/index.js +18 -1
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +84 -32
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -5
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +7 -5
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
- package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
- package/dist/model-fit/floor.d.ts +31 -0
- package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
- package/dist/model-fit/kv-cache.d.ts +23 -0
- package/dist/model-fit/kv-cache.js +85 -0
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
- package/dist/model-fit/platform.d.ts +14 -0
- package/dist/model-fit/platform.js +26 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
- package/dist/model-fit/request/create.js +61 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/model-fit/request/engines/audiogen.js +63 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
- package/dist/model-fit/request/engines/llama.js +44 -0
- package/dist/model-fit/request/engines/nmt.d.ts +8 -0
- package/dist/model-fit/request/engines/nmt.js +11 -0
- package/dist/model-fit/request/engines/ocr.d.ts +9 -0
- package/dist/model-fit/request/engines/ocr.js +13 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1072 -815
- package/dist/models/registry/models.js +3704 -3405
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +406 -3
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
- package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
- package/dist/plugins/builtin/index.d.ts +15 -12
- package/dist/plugins/builtin/index.js +15 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
- package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
- package/dist/plugins/ops/kv-cache-utils.js +25 -33
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +33 -19
- package/dist/plugins/ops/transcribe.js +18 -2
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +3 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +9 -1
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +22 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +7 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +106 -43
- package/dist/schemas/assess-model-fit.js +69 -32
- package/dist/schemas/audio-gen.d.ts +199 -19
- package/dist/schemas/audio-gen.js +183 -51
- package/dist/schemas/batch-completion-stream.d.ts +210 -36
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +3 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +24 -3
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +1465 -162
- package/dist/schemas/common.js +37 -31
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +243 -39
- package/dist/schemas/completion-stream.js +100 -29
- package/dist/schemas/config.d.ts +1326 -18
- package/dist/schemas/config.js +30 -8
- package/dist/schemas/decision.d.ts +610 -0
- package/dist/schemas/decision.js +219 -0
- package/dist/schemas/download-asset.d.ts +10 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +9 -1
- package/dist/schemas/engine-addon-map.js +26 -5
- package/dist/schemas/errors.d.ts +2 -0
- package/dist/schemas/errors.js +11 -1
- package/dist/schemas/index.d.ts +51 -48
- package/dist/schemas/index.js +50 -47
- package/dist/schemas/llamacpp-config.d.ts +222 -6
- package/dist/schemas/llamacpp-config.js +129 -7
- package/dist/schemas/load-model.d.ts +7960 -1492
- package/dist/schemas/load-model.js +122 -10
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +428 -9
- package/dist/schemas/model-config-schemas.js +16 -11
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +4 -3
- package/dist/schemas/model-resource-profile.d.ts +4 -4
- package/dist/schemas/model-resource-profile.js +2 -5
- package/dist/schemas/model-src-utils.d.ts +14 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +14 -1
- package/dist/schemas/model-types.js +4 -0
- package/dist/schemas/ocr.d.ts +76 -0
- package/dist/schemas/ocr.js +43 -1
- package/dist/schemas/plugin.d.ts +16 -4
- package/dist/schemas/plugin.js +12 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +34 -0
- package/dist/schemas/registry.js +5 -1
- package/dist/schemas/reload-config.d.ts +14 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +82 -0
- package/dist/schemas/sdcpp-config.js +41 -22
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +463 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +20 -0
- package/dist/schemas/transcription-config.js +54 -4
- package/dist/schemas/transcription.d.ts +47 -5
- package/dist/schemas/transcription.js +52 -5
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +40 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +3 -2
- package/dist/surface.js +3 -2
- package/dist/utils/addon-responses.d.ts +3 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +13 -9
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +2 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +68 -51
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -28
- package/dist/resources/model-fit/types.d.ts +0 -120
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
|
@@ -1,13 +1,7 @@
|
|
|
1
|
-
import { getEngineLogger } from '
|
|
2
|
-
import { classifyFit } from '
|
|
3
|
-
import { createFitRequest } from '
|
|
4
|
-
import { runFit as runFitDefault } from '
|
|
5
|
-
/**
|
|
6
|
-
* Shorter than the supervisor's own 60s default: this check sits in front of a
|
|
7
|
-
* real load, so a wedged child must not hold the load for a full minute. Every
|
|
8
|
-
* expiry is `unknown` and the load continues.
|
|
9
|
-
*/
|
|
10
|
-
const ADVISORY_FIT_TIMEOUT_MS = 30_000;
|
|
1
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
2
|
+
import { classifyFit } from '../model-fit/classify.js';
|
|
3
|
+
import { createFitRequest } from '../model-fit/request/create.js';
|
|
4
|
+
import { runFit as runFitDefault } from '../model-fit/run/run.js';
|
|
11
5
|
/**
|
|
12
6
|
* The base every engine's fitter withholds by default. Made explicit here
|
|
13
7
|
* because the resident-model reserve is added to it: setting a margin at all
|
|
@@ -24,30 +18,6 @@ const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
|
|
|
24
18
|
*/
|
|
25
19
|
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v2';
|
|
26
20
|
const BYTES_PER_MIB = 1024 * 1024;
|
|
27
|
-
const DISABLED_VALUES = new Set(['0', 'false', 'off', 'no']);
|
|
28
|
-
/**
|
|
29
|
-
* On by default; `QVAC_ADVISORY_MODEL_FIT=0` (or `false`/`off`/`no`) is the
|
|
30
|
-
* operator opt-out — the escape hatch for a load-heavy startup path or a
|
|
31
|
-
* runtime whose operator would rather not pay for it. Any other value, including
|
|
32
|
-
* unset, leaves the check on.
|
|
33
|
-
*
|
|
34
|
-
* The worker environment and the mobile runtime flag are imported lazily. Both
|
|
35
|
-
* modules reach Bare-only bindings, and resolving them eagerly would make this
|
|
36
|
-
* orchestration untestable outside a Bare runtime.
|
|
37
|
-
*/
|
|
38
|
-
async function resolveEnabled(explicit) {
|
|
39
|
-
if (explicit !== undefined)
|
|
40
|
-
return explicit;
|
|
41
|
-
const { getValidatedEnv } = await import('../../../runtime/env.js');
|
|
42
|
-
const value = getValidatedEnv().QVAC_ADVISORY_MODEL_FIT;
|
|
43
|
-
return value === undefined || !DISABLED_VALUES.has(value.toLowerCase());
|
|
44
|
-
}
|
|
45
|
-
async function resolveMobile(explicit) {
|
|
46
|
-
if (explicit !== undefined)
|
|
47
|
-
return explicit;
|
|
48
|
-
const { isMobile } = await import('../../../runtime/state.js');
|
|
49
|
-
return isMobile();
|
|
50
|
-
}
|
|
51
21
|
/**
|
|
52
22
|
* Sums the on-disk weight sizes of every model currently registered in this
|
|
53
23
|
* worker. The fallback for a host whose memory sample is unavailable, where
|
|
@@ -59,7 +29,7 @@ async function resolveMobile(explicit) {
|
|
|
59
29
|
*/
|
|
60
30
|
async function defaultResidentModelBytes() {
|
|
61
31
|
const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
|
|
62
|
-
import('
|
|
32
|
+
import('../runtime/model-registry.js'),
|
|
63
33
|
import('bare-fs')
|
|
64
34
|
]);
|
|
65
35
|
let bytes = 0;
|
|
@@ -97,9 +67,9 @@ function mib(bytes) {
|
|
|
97
67
|
*/
|
|
98
68
|
async function defaultAvailableSystemBytes() {
|
|
99
69
|
const [{ getResourceCollector }, { resolveBasis, resolveBudget }, { detectPlatform }] = await Promise.all([
|
|
100
|
-
import('
|
|
101
|
-
import('../assess.js'),
|
|
102
|
-
import('../platform.js')
|
|
70
|
+
import('../resources/instance.js'),
|
|
71
|
+
import('../model-fit/assess.js'),
|
|
72
|
+
import('../model-fit/platform.js')
|
|
103
73
|
]);
|
|
104
74
|
const collector = getResourceCollector();
|
|
105
75
|
if (!collector)
|
|
@@ -117,15 +87,9 @@ async function defaultAvailableSystemBytes() {
|
|
|
117
87
|
* is system RAM, which is what `boundBySystemMemory` decides.
|
|
118
88
|
*/
|
|
119
89
|
async function deviceBytesCountAgainstSystem() {
|
|
120
|
-
const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([
|
|
121
|
-
import('../../instance.js'),
|
|
122
|
-
import('../assess.js'),
|
|
123
|
-
import('../platform.js')
|
|
124
|
-
]);
|
|
90
|
+
const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([import('../resources/instance.js'), import('../model-fit/assess.js'), import('../model-fit/platform.js')]);
|
|
125
91
|
const collector = getResourceCollector();
|
|
126
|
-
// Unknown counts them
|
|
127
|
-
// this check exists to stop, and refusing one that would have fitted is
|
|
128
|
-
// advisory only.
|
|
92
|
+
// Unknown counts them, which errs toward refusing.
|
|
129
93
|
if (!collector)
|
|
130
94
|
return true;
|
|
131
95
|
const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
@@ -182,13 +146,13 @@ function report(logger, input, outcome) {
|
|
|
182
146
|
const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
|
|
183
147
|
if (outcome.verdict === 'fit') {
|
|
184
148
|
const plan = outcome.plan;
|
|
185
|
-
logger.info(`${prefix} projected to fit
|
|
149
|
+
logger.info(`${prefix} projected to fit${plan === undefined
|
|
186
150
|
? footprint(outcome)
|
|
187
151
|
: ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
|
|
188
152
|
return;
|
|
189
153
|
}
|
|
190
154
|
if (outcome.verdict === 'does-not-fit') {
|
|
191
|
-
logger.warn(`${prefix} projected not to fit
|
|
155
|
+
logger.warn(`${prefix} projected not to fit${footprint(outcome)}: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`);
|
|
192
156
|
return;
|
|
193
157
|
}
|
|
194
158
|
// `unsupported-load` covers every load this check refuses up front — a model
|
|
@@ -207,28 +171,27 @@ function report(logger, input, outcome) {
|
|
|
207
171
|
* Runs the advisory fit check for a load that is about to start, against the
|
|
208
172
|
* fitter belonging to the engine that will run it.
|
|
209
173
|
*
|
|
210
|
-
* Fail-open by construction: an unsupported shape, a
|
|
211
|
-
*
|
|
212
|
-
* and the caller proceeds with the ordinary load path.
|
|
213
|
-
* throws and never rejects.
|
|
174
|
+
* Fail-open by construction: an unsupported shape, a fitter that crashed,
|
|
175
|
+
* threw or did not return in time, and an unexpected internal error all
|
|
176
|
+
* resolve to `unknown` and the caller proceeds with the ordinary load path.
|
|
177
|
+
* This function never throws and never rejects.
|
|
214
178
|
*/
|
|
215
179
|
export async function runAdvisoryFitCheck(input, options = {}) {
|
|
216
180
|
let logger = options.logger;
|
|
217
181
|
try {
|
|
218
182
|
logger ??= getEngineLogger();
|
|
219
|
-
if (!(await resolveEnabled(options.enabled)))
|
|
220
|
-
return unknown('disabled');
|
|
221
|
-
const mobile = await resolveMobile(options.mobile);
|
|
222
183
|
const availableBytes = await (options.availableSystemBytes ?? defaultAvailableSystemBytes)();
|
|
223
|
-
const residentBytes = await (options.residentModelBytes ?? defaultResidentModelBytes)()
|
|
184
|
+
const residentBytes = (await (options.residentModelBytes ?? defaultResidentModelBytes)()) +
|
|
185
|
+
(options.extraResidentBytes ?? 0);
|
|
224
186
|
const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
|
|
225
|
-
// Always sent, even with a zero reserve
|
|
226
|
-
//
|
|
227
|
-
// diverge silently if an engine default moves.
|
|
187
|
+
// Always sent, even with a zero reserve, so the base margin has one source
|
|
188
|
+
// and an engine default never decides it.
|
|
228
189
|
const plan = createFitRequest({
|
|
229
190
|
modelType: input.modelType,
|
|
230
191
|
modelPath: input.modelPath,
|
|
231
192
|
modelConfig: input.modelConfig,
|
|
193
|
+
transcriptionWorkload: input.transcriptionWorkload,
|
|
194
|
+
ttsWorkload: input.ttsWorkload,
|
|
232
195
|
artifacts: input.artifacts,
|
|
233
196
|
isShardedModel: input.isShardedModel,
|
|
234
197
|
marginBytes: (ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB) * BYTES_PER_MIB
|
|
@@ -238,11 +201,7 @@ export async function runAdvisoryFitCheck(input, options = {}) {
|
|
|
238
201
|
report(logger, input, outcome);
|
|
239
202
|
return outcome;
|
|
240
203
|
}
|
|
241
|
-
const result = await (options.runFit ?? runFitDefault)(plan.probe, {
|
|
242
|
-
mobile,
|
|
243
|
-
timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
|
|
244
|
-
...(options.signal !== undefined && { signal: options.signal })
|
|
245
|
-
});
|
|
204
|
+
const result = await (options.runFit ?? runFitDefault)(plan.probe, options.signal !== undefined ? { signal: options.signal } : {});
|
|
246
205
|
const outcome = result.status === 'completed'
|
|
247
206
|
? withMachineBudget(classifyFit(result.probe, PROVENANCE), availableBytes, await (options.countsDeviceRows ?? deviceBytesCountAgainstSystem)())
|
|
248
207
|
: unknown(result.reason, result.message);
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Platforms this assessment covers. A platform absent from this list assesses
|
|
3
|
+
* as `unknown`, because nothing fixes the memory budget it would be judged
|
|
4
|
+
* against.
|
|
5
|
+
*/
|
|
6
|
+
export type ModelFitPlatform = 'darwin-arm64' | 'darwin-x64' | 'linux-arm64' | 'linux-x64' | 'win32-x64' | 'win32-arm64' | 'android-arm64' | 'ios-arm64';
|
|
7
|
+
/**
|
|
8
|
+
* Maps the runtime's platform and architecture onto a platform this assessment
|
|
9
|
+
* covers.
|
|
10
|
+
*
|
|
11
|
+
* @returns `undefined` for any pair outside that set, which assesses as
|
|
12
|
+
* `unknown`.
|
|
13
|
+
*/
|
|
14
|
+
export declare function detectPlatform(): ModelFitPlatform | undefined;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import os from 'bare-os';
|
|
2
|
+
/**
|
|
3
|
+
* Maps the runtime's platform and architecture onto a platform this assessment
|
|
4
|
+
* covers.
|
|
5
|
+
*
|
|
6
|
+
* @returns `undefined` for any pair outside that set, which assesses as
|
|
7
|
+
* `unknown`.
|
|
8
|
+
*/
|
|
9
|
+
export function detectPlatform() {
|
|
10
|
+
const key = `${os.platform()}-${os.arch()}`;
|
|
11
|
+
switch (key) {
|
|
12
|
+
case 'darwin-arm64':
|
|
13
|
+
case 'darwin-x64':
|
|
14
|
+
case 'linux-arm64':
|
|
15
|
+
case 'linux-x64':
|
|
16
|
+
case 'win32-x64':
|
|
17
|
+
// No engine addon is built for win32-arm64 (`@qvac/llm-llamacpp/prebuilds`
|
|
18
|
+
// has no such target), so no load on it reaches a fitter.
|
|
19
|
+
case 'win32-arm64':
|
|
20
|
+
case 'android-arm64':
|
|
21
|
+
case 'ios-arm64':
|
|
22
|
+
return key;
|
|
23
|
+
default:
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import type
|
|
1
|
+
import type { TtsFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
import { type CanonicalModelType, type TranscriptionFitWorkload } from '../../schemas/index.js';
|
|
3
|
+
import type { FitProbeRequest } from '../../model-fit/run/engine.js';
|
|
3
4
|
export type FitRequestPlan = {
|
|
4
5
|
supported: true;
|
|
5
6
|
probe: FitProbeRequest;
|
|
@@ -8,9 +9,11 @@ export type FitRequestPlan = {
|
|
|
8
9
|
detail: string;
|
|
9
10
|
};
|
|
10
11
|
export interface CreateFitRequestParams {
|
|
12
|
+
ttsWorkload?: TtsFitWorkload | undefined;
|
|
11
13
|
modelType: CanonicalModelType;
|
|
12
14
|
modelPath: string;
|
|
13
15
|
modelConfig: unknown;
|
|
16
|
+
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
14
17
|
artifacts?: Record<string, string> | undefined;
|
|
15
18
|
isShardedModel: boolean;
|
|
16
19
|
/** Free memory that must remain for the projection to count as fitting. */
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { ModelType } from '../../schemas/index.js';
|
|
2
|
+
import { createBciFitRequest, createMossTranscribeFitRequest, createParakeetFitRequest, createWhisperFitRequest } from '../../model-fit/request/engines/asr.js';
|
|
3
|
+
import { createAudiogenFitRequest } from '../../model-fit/request/engines/audiogen.js';
|
|
4
|
+
import { createDiffusionFitRequest } from '../../model-fit/request/engines/diffusion.js';
|
|
5
|
+
import { createLlamaFitRequest, llamaLoadKindFor } from '../../model-fit/request/engines/llama.js';
|
|
6
|
+
import { createNmtFitRequest } from '../../model-fit/request/engines/nmt.js';
|
|
7
|
+
import { createOcrFitRequest } from '../../model-fit/request/engines/ocr.js';
|
|
8
|
+
import { createTtsFitRequest } from '../../model-fit/request/engines/tts.js';
|
|
9
|
+
/**
|
|
10
|
+
* Builds a fit request from the same resolved config and artifacts the real
|
|
11
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
12
|
+
*
|
|
13
|
+
* Structural shapes the SDK owns — a load with no fitter behind it, a companion
|
|
14
|
+
* set that did not resolve, a setting whose effect on the projection is unknown
|
|
15
|
+
* — are refused here so no engine is asked a question it cannot answer.
|
|
16
|
+
* Value-level policy stays inside each engine's own fitter.
|
|
17
|
+
*/
|
|
18
|
+
export function createFitRequest(params) {
|
|
19
|
+
const { modelType, modelPath, modelConfig, artifacts, marginBytes } = params;
|
|
20
|
+
if (params.transcriptionWorkload !== undefined && modelType !== ModelType.mossTranscribe) {
|
|
21
|
+
return {
|
|
22
|
+
supported: false,
|
|
23
|
+
detail: 'transcriptionWorkload is supported only for MOSS transcription'
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
// Every engine reads whole files rather than a shard set, so a split model is
|
|
27
|
+
// not a shape any of them can be pointed at.
|
|
28
|
+
if (params.isShardedModel) {
|
|
29
|
+
return { supported: false, detail: 'sharded models are not representable' };
|
|
30
|
+
}
|
|
31
|
+
const loadKind = llamaLoadKindFor(modelType);
|
|
32
|
+
if (loadKind !== undefined) {
|
|
33
|
+
return createLlamaFitRequest({ loadKind, modelPath, modelConfig, artifacts, marginBytes });
|
|
34
|
+
}
|
|
35
|
+
const common = { modelPath, modelConfig, artifacts, marginBytes };
|
|
36
|
+
switch (modelType) {
|
|
37
|
+
case ModelType.mossTranscribe:
|
|
38
|
+
return createMossTranscribeFitRequest({
|
|
39
|
+
...common,
|
|
40
|
+
transcriptionWorkload: params.transcriptionWorkload
|
|
41
|
+
});
|
|
42
|
+
case ModelType.whispercppTranscription:
|
|
43
|
+
return createWhisperFitRequest(common);
|
|
44
|
+
case ModelType.parakeetTranscription:
|
|
45
|
+
return createParakeetFitRequest(common);
|
|
46
|
+
case ModelType.bciWhispercppTranscription:
|
|
47
|
+
return createBciFitRequest(common);
|
|
48
|
+
case ModelType.ttsGgml:
|
|
49
|
+
return createTtsFitRequest({ ...common, ttsWorkload: params.ttsWorkload });
|
|
50
|
+
case ModelType.audiogenGgml:
|
|
51
|
+
return createAudiogenFitRequest(common);
|
|
52
|
+
case ModelType.sdcppGeneration:
|
|
53
|
+
return createDiffusionFitRequest({ modelPath, modelConfig, artifacts });
|
|
54
|
+
case ModelType.ggmlOcr:
|
|
55
|
+
return createOcrFitRequest(common);
|
|
56
|
+
case ModelType.nmtcppTranslation:
|
|
57
|
+
return createNmtFitRequest(common);
|
|
58
|
+
default:
|
|
59
|
+
return { supported: false, detail: `no fitter for model type: ${modelType}` };
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { TranscriptionFitWorkload } from '../../../schemas/index.js';
|
|
2
|
+
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
2
3
|
export interface AsrFitRequestParams {
|
|
3
4
|
modelPath: string;
|
|
4
5
|
modelConfig: unknown;
|
|
5
6
|
artifacts?: Record<string, string> | undefined;
|
|
6
7
|
marginBytes?: number | undefined;
|
|
7
8
|
}
|
|
9
|
+
export declare function createMossTranscribeFitRequest(params: AsrFitRequestParams & {
|
|
10
|
+
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
11
|
+
}): FitRequestPlan;
|
|
8
12
|
export declare function createWhisperFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
9
13
|
export declare function createParakeetFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
10
14
|
export declare function createBciFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
@@ -1,4 +1,27 @@
|
|
|
1
|
-
import { gpuLayersFromGate } from '
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/request/gpu.js';
|
|
2
|
+
export function createMossTranscribeFitRequest(params) {
|
|
3
|
+
if (params.transcriptionWorkload === undefined) {
|
|
4
|
+
return {
|
|
5
|
+
supported: false,
|
|
6
|
+
detail: 'MOSS fit requires transcriptionWorkload with explicit audioSeconds'
|
|
7
|
+
};
|
|
8
|
+
}
|
|
9
|
+
const config = (params.modelConfig ?? {});
|
|
10
|
+
const workload = params.transcriptionWorkload;
|
|
11
|
+
const request = {
|
|
12
|
+
engine: 'moss-transcribe',
|
|
13
|
+
modelPath: params.modelPath,
|
|
14
|
+
audioSeconds: workload.audioSeconds,
|
|
15
|
+
...(workload.prompt !== undefined && { prompt: workload.prompt }),
|
|
16
|
+
...(workload.hotwords !== undefined && { hotwords: workload.hotwords }),
|
|
17
|
+
...(workload.maxNewTokens !== undefined && { maxNewTokens: workload.maxNewTokens }),
|
|
18
|
+
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
19
|
+
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
20
|
+
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
|
|
21
|
+
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
22
|
+
};
|
|
23
|
+
return { supported: true, probe: { engine: 'asr-ggml', request } };
|
|
24
|
+
}
|
|
2
25
|
/**
|
|
3
26
|
* The longest single transcribe the projection has to cover. Whisper's KV cache
|
|
4
27
|
* and decode graph are sized from it; unset leaves the engine's own default,
|
|
@@ -69,6 +92,14 @@ export function createParakeetFitRequest(params) {
|
|
|
69
92
|
modelPath: params.modelPath,
|
|
70
93
|
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
71
94
|
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
95
|
+
// The long-form window bounds the offline encoder graph the projection
|
|
96
|
+
// sizes, so an explicit window (or the one-pass sentinel) reaches the fit.
|
|
97
|
+
...(config.longFormWindowFrames !== undefined && {
|
|
98
|
+
longFormWindowFrames: config.longFormWindowFrames
|
|
99
|
+
}),
|
|
100
|
+
...(config.longFormContextFrames !== undefined && {
|
|
101
|
+
longFormContextFrames: config.longFormContextFrames
|
|
102
|
+
}),
|
|
72
103
|
// Only a streaming load holds a live session, so outside one the chunk
|
|
73
104
|
// cadence describes no allocation the projection should cover.
|
|
74
105
|
...(config.streaming === true &&
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/request/gpu.js';
|
|
2
|
+
/**
|
|
3
|
+
* `@qvac/audiogen-ggml` is an optional peer dependency, so its value exports
|
|
4
|
+
* are unavailable to a host that ships without it. The engine discriminator is
|
|
5
|
+
* matched by its literal rather than imported.
|
|
6
|
+
*/
|
|
7
|
+
const MINIMAX = 'minimax';
|
|
8
|
+
/**
|
|
9
|
+
* MiniMax-Music3 is fitted from its LM and synthesis pair. The load config
|
|
10
|
+
* carries no frame budget, so the fitter projects the default generation
|
|
11
|
+
* `run()` falls back to (300 frames, 12 s).
|
|
12
|
+
*
|
|
13
|
+
* `device` is passed through as the load takes it. Without one, the GPU switch
|
|
14
|
+
* maps to `auto` as it does on load: the fitter reads a positive layer count
|
|
15
|
+
* that way. The plugin refuses MiniMax on mobile before this is reached, and
|
|
16
|
+
* mobile builds carry no MiniMax fitter.
|
|
17
|
+
*/
|
|
18
|
+
function minimaxFitRequest(config, artifacts, marginBytes) {
|
|
19
|
+
const lmPath = artifacts['lmModelPath'];
|
|
20
|
+
const synthPath = artifacts['synthModelPath'];
|
|
21
|
+
if (lmPath === undefined || synthPath === undefined) {
|
|
22
|
+
return { supported: false, detail: 'the minimax model pair is not resolved' };
|
|
23
|
+
}
|
|
24
|
+
const request = {
|
|
25
|
+
engine: MINIMAX,
|
|
26
|
+
lmPath,
|
|
27
|
+
synthPath,
|
|
28
|
+
...(config.device !== undefined
|
|
29
|
+
? { device: config.device }
|
|
30
|
+
: { gpuLayers: gpuLayersFromGate(config.useGPU) }),
|
|
31
|
+
...(config.threads !== undefined && { threads: config.threads }),
|
|
32
|
+
...(marginBytes !== undefined && { marginBytes })
|
|
33
|
+
};
|
|
34
|
+
return { supported: true, probe: { engine: 'audiogen-ggml', request } };
|
|
35
|
+
}
|
|
36
|
+
export function createAudiogenFitRequest(params) {
|
|
37
|
+
const config = (params.modelConfig ?? {});
|
|
38
|
+
const artifacts = params.artifacts ?? {};
|
|
39
|
+
if (config.engine === MINIMAX) {
|
|
40
|
+
return minimaxFitRequest(config, artifacts, params.marginBytes);
|
|
41
|
+
}
|
|
42
|
+
const textEncoderPath = artifacts['textEncModelPath'];
|
|
43
|
+
const lmPath = artifacts['lmModelPath'];
|
|
44
|
+
const ditPath = artifacts['ditModelPath'];
|
|
45
|
+
const vaePath = artifacts['vaeModelPath'];
|
|
46
|
+
if (textEncoderPath === undefined ||
|
|
47
|
+
lmPath === undefined ||
|
|
48
|
+
ditPath === undefined ||
|
|
49
|
+
vaePath === undefined) {
|
|
50
|
+
return { supported: false, detail: 'the ace-step stage checkpoints are not all resolved' };
|
|
51
|
+
}
|
|
52
|
+
const layers = 'nGpuLayers' in config ? config.nGpuLayers : undefined;
|
|
53
|
+
const request = {
|
|
54
|
+
textEncoderPath,
|
|
55
|
+
lmPath,
|
|
56
|
+
ditPath,
|
|
57
|
+
vaePath,
|
|
58
|
+
gpuLayers: gpuLayersFromGate(config.useGPU, layers),
|
|
59
|
+
...(config.threads !== undefined && { threads: config.threads }),
|
|
60
|
+
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
61
|
+
};
|
|
62
|
+
return { supported: true, probe: { engine: 'audiogen-ggml', request } };
|
|
63
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { flattenUpscalerKeys, toEsrganAddonConfig } from '../../../plugins/builtin/sdcpp-generation/upscaler-config.js';
|
|
2
|
+
import { splitWorldConfig } from '../../../plugins/builtin/sdcpp-generation/world-config.js';
|
|
3
|
+
/**
|
|
4
|
+
* Frames to project a video load at. The fitter reads one frame when nothing
|
|
5
|
+
* is passed, which projects image generation; `run` generates 33. The frame
|
|
6
|
+
* count is a generation parameter, and this assessment sees load config alone.
|
|
7
|
+
*/
|
|
8
|
+
const DEFAULT_VIDEO_FRAMES = 33;
|
|
9
|
+
export function createDiffusionFitRequest(params) {
|
|
10
|
+
const config = (params.modelConfig ?? {});
|
|
11
|
+
const artifacts = params.artifacts ?? {};
|
|
12
|
+
if (config.mode === 'world') {
|
|
13
|
+
const taehv = artifacts['taehvModelPath'];
|
|
14
|
+
const scene = artifacts['seedScenePath'];
|
|
15
|
+
if (taehv === undefined || scene === undefined) {
|
|
16
|
+
return { supported: false, detail: 'ABot fit requires a resolved decoder and scene pack' };
|
|
17
|
+
}
|
|
18
|
+
const { fitSteps, session } = splitWorldConfig(config.world);
|
|
19
|
+
return {
|
|
20
|
+
supported: true,
|
|
21
|
+
probe: {
|
|
22
|
+
engine: 'diffusion-cpp',
|
|
23
|
+
request: {
|
|
24
|
+
mode: 'world',
|
|
25
|
+
files: { model: params.modelPath, taehv, scene },
|
|
26
|
+
config: session,
|
|
27
|
+
workload: { walkSteps: fitSteps }
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
if (config.mode === 'upscale') {
|
|
33
|
+
return {
|
|
34
|
+
supported: true,
|
|
35
|
+
probe: {
|
|
36
|
+
engine: 'diffusion-cpp',
|
|
37
|
+
request: {
|
|
38
|
+
mode: 'upscale',
|
|
39
|
+
files: { esrgan: params.modelPath },
|
|
40
|
+
config: toEsrganAddonConfig(config),
|
|
41
|
+
workload: { upscaleRepeats: 1 }
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const upscaler = config.mode === 'video' ? undefined : config.upscaler;
|
|
47
|
+
const esrgan = upscaler === undefined ? undefined : artifacts['esrganModelPath'];
|
|
48
|
+
if (upscaler !== undefined && esrgan === undefined) {
|
|
49
|
+
return { supported: false, detail: 'the configured ESRGAN checkpoint is not resolved' };
|
|
50
|
+
}
|
|
51
|
+
const files = {
|
|
52
|
+
model: params.modelPath,
|
|
53
|
+
...(esrgan !== undefined && { esrgan }),
|
|
54
|
+
...(artifacts['clipLModelPath'] !== undefined && { clipL: artifacts['clipLModelPath'] }),
|
|
55
|
+
...(artifacts['clipGModelPath'] !== undefined && { clipG: artifacts['clipGModelPath'] }),
|
|
56
|
+
...(artifacts['t5XxlModelPath'] !== undefined && { t5Xxl: artifacts['t5XxlModelPath'] }),
|
|
57
|
+
...(artifacts['llmModelPath'] !== undefined && { llm: artifacts['llmModelPath'] }),
|
|
58
|
+
...(artifacts['projectionModelPath'] !== undefined && {
|
|
59
|
+
llmVision: artifacts['projectionModelPath']
|
|
60
|
+
}),
|
|
61
|
+
...(artifacts['vaeModelPath'] !== undefined && { vae: artifacts['vaeModelPath'] }),
|
|
62
|
+
...(artifacts['uncondModelPath'] !== undefined && {
|
|
63
|
+
uncondModel: artifacts['uncondModelPath']
|
|
64
|
+
}),
|
|
65
|
+
...(artifacts['clipVisionModelPath'] !== undefined && {
|
|
66
|
+
clipVision: artifacts['clipVisionModelPath']
|
|
67
|
+
}),
|
|
68
|
+
...(artifacts['audioVaeModelPath'] !== undefined && {
|
|
69
|
+
audioVae: artifacts['audioVaeModelPath']
|
|
70
|
+
}),
|
|
71
|
+
...(artifacts['embeddingsConnectorsModelPath'] !== undefined && {
|
|
72
|
+
embeddingsConnectors: artifacts['embeddingsConnectorsModelPath']
|
|
73
|
+
})
|
|
74
|
+
};
|
|
75
|
+
// `mode` and `upscaler` are this SDK's own keys; the engine config has no
|
|
76
|
+
// field for either.
|
|
77
|
+
const { mode: _mode, upscaler: _upscaler, ...engineConfig } = config;
|
|
78
|
+
return {
|
|
79
|
+
supported: true,
|
|
80
|
+
probe: {
|
|
81
|
+
engine: 'diffusion-cpp',
|
|
82
|
+
request: {
|
|
83
|
+
files,
|
|
84
|
+
config: { ...engineConfig, ...flattenUpscalerKeys(upscaler) },
|
|
85
|
+
workload: {
|
|
86
|
+
...(config.vae_tiling !== undefined && { vaeTiling: config.vae_tiling }),
|
|
87
|
+
...(config.mode === 'video' && { videoFrames: DEFAULT_VIDEO_FRAMES }),
|
|
88
|
+
...(upscaler !== undefined && { upscaleRepeats: 1 })
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type CanonicalModelType } from '
|
|
2
|
-
import type { FitRequestPlan } from '
|
|
1
|
+
import { type CanonicalModelType } from '../../../schemas/index.js';
|
|
2
|
+
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
3
3
|
export type LlamaLoadKind = 'completion' | 'embedding';
|
|
4
4
|
export declare function llamaLoadKindFor(modelType: CanonicalModelType): LlamaLoadKind | undefined;
|
|
5
5
|
export interface LlamaFitRequestParams {
|
|
@@ -7,7 +7,6 @@ export interface LlamaFitRequestParams {
|
|
|
7
7
|
modelPath: string;
|
|
8
8
|
modelConfig: unknown;
|
|
9
9
|
artifacts?: Record<string, string> | undefined;
|
|
10
|
-
isShardedModel: boolean;
|
|
11
10
|
marginBytes?: number | undefined;
|
|
12
11
|
}
|
|
13
12
|
export declare function createLlamaFitRequest(params: LlamaFitRequestParams): FitRequestPlan;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { ModelType } from '../../../schemas/index.js';
|
|
2
|
+
import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
|
|
3
|
+
import { transformEmbedConfig } from '../../../plugins/builtin/llamacpp-embedding/transform.js';
|
|
4
|
+
export function llamaLoadKindFor(modelType) {
|
|
5
|
+
if (modelType === ModelType.llamacppCompletion)
|
|
6
|
+
return 'completion';
|
|
7
|
+
if (modelType === ModelType.llamacppEmbedding)
|
|
8
|
+
return 'embedding';
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
export function createLlamaFitRequest(params) {
|
|
12
|
+
const { loadKind } = params;
|
|
13
|
+
if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
|
|
14
|
+
return { supported: false, detail: 'multimodal projection loads are not representable' };
|
|
15
|
+
}
|
|
16
|
+
const modelConfig = (params.modelConfig ?? {});
|
|
17
|
+
// The assessment budgets against this machine's devices, so a load spread
|
|
18
|
+
// over remote GPUs is refused rather than measured against the wrong ones.
|
|
19
|
+
if (modelConfig['rpc-servers'] !== undefined) {
|
|
20
|
+
return { supported: false, detail: 'unsupported load setting: rpc-servers' };
|
|
21
|
+
}
|
|
22
|
+
const request = {
|
|
23
|
+
modelPath: params.modelPath,
|
|
24
|
+
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
25
|
+
};
|
|
26
|
+
if (loadKind === 'completion') {
|
|
27
|
+
// The addon's fitter rejects these as unknown arguments, and none changes
|
|
28
|
+
// what it projects: the image limit gates images at inference, and the
|
|
29
|
+
// prompt-cache keys size host-RAM and disk state the fitter does not model.
|
|
30
|
+
const { 'image-max-megapixels': _imageMaxMegapixels, cache_checkpoints: _cacheCheckpoints, cache_checkpoints_max_bytes: _cacheCheckpointsMaxBytes, cache_checkpoint_storage: _cacheCheckpointStorage, cache_checkpoint_dir: _cacheCheckpointDir, cache_ram_mib: _cacheRamMib, ...fitConfig } = modelConfig;
|
|
31
|
+
const config = transformLlmConfig(fitConfig);
|
|
32
|
+
// The fit reads the draft model (DFlash model or external MTP head) like the load does.
|
|
33
|
+
const draftModelPath = params.artifacts?.['draftModelPath'];
|
|
34
|
+
if (draftModelPath)
|
|
35
|
+
config['spec_draft_model'] = draftModelPath;
|
|
36
|
+
const llm = { ...request, config };
|
|
37
|
+
return { supported: true, probe: { engine: 'llm-llamacpp', request: llm } };
|
|
38
|
+
}
|
|
39
|
+
const embed = {
|
|
40
|
+
...request,
|
|
41
|
+
config: transformEmbedConfig(modelConfig)
|
|
42
|
+
};
|
|
43
|
+
return { supported: true, probe: { engine: 'embed-llamacpp', request: embed } };
|
|
44
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
2
|
+
export interface NmtFitRequestParams {
|
|
3
|
+
modelPath: string;
|
|
4
|
+
modelConfig: unknown;
|
|
5
|
+
artifacts?: Record<string, string> | undefined;
|
|
6
|
+
marginBytes?: number | undefined;
|
|
7
|
+
}
|
|
8
|
+
export declare function createNmtFitRequest(params: NmtFitRequestParams): FitRequestPlan;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { nmtConfigFor, nmtFilesFor } from '../../../plugins/builtin/nmtcpp-translation/load-files.js';
|
|
2
|
+
export function createNmtFitRequest(params) {
|
|
3
|
+
const config = (params.modelConfig ?? {});
|
|
4
|
+
const files = nmtFilesFor(params.modelPath, params.artifacts ?? {});
|
|
5
|
+
const request = {
|
|
6
|
+
files,
|
|
7
|
+
config: nmtConfigFor(config, files.pivotModel),
|
|
8
|
+
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
9
|
+
};
|
|
10
|
+
return { supported: true, probe: { engine: 'translation-nmtcpp', request } };
|
|
11
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
2
|
+
export interface OcrFitRequestParams {
|
|
3
|
+
/** The recognizer, which is the load's primary model. */
|
|
4
|
+
modelPath: string;
|
|
5
|
+
modelConfig: unknown;
|
|
6
|
+
artifacts?: Record<string, string> | undefined;
|
|
7
|
+
marginBytes?: number | undefined;
|
|
8
|
+
}
|
|
9
|
+
export declare function createOcrFitRequest(params: OcrFitRequestParams): FitRequestPlan;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { ocrParamsFor } from '../../../plugins/builtin/ggml-ocr/params.js';
|
|
2
|
+
export function createOcrFitRequest(params) {
|
|
3
|
+
const detectorPath = params.artifacts?.['detectorModelPath'];
|
|
4
|
+
if (detectorPath === undefined) {
|
|
5
|
+
return { supported: false, detail: 'the ocr detector is not resolved' };
|
|
6
|
+
}
|
|
7
|
+
const config = (params.modelConfig ?? {});
|
|
8
|
+
const request = {
|
|
9
|
+
...ocrParamsFor(detectorPath, params.modelPath, config),
|
|
10
|
+
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
11
|
+
};
|
|
12
|
+
return { supported: true, probe: { engine: 'ocr-ggml', request } };
|
|
13
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { TtsFitWorkload } from '../../../schemas/assess-model-fit.js';
|
|
2
|
+
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
2
3
|
export interface TtsFitRequestParams {
|
|
4
|
+
ttsWorkload?: TtsFitWorkload | undefined;
|
|
3
5
|
modelPath: string;
|
|
4
6
|
modelConfig: unknown;
|
|
5
7
|
artifacts?: Record<string, string> | undefined;
|