@qvac/inference 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +89 -41
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +5 -2
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -2
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +39 -37
- package/dist/api/index.js +38 -36
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +6 -5
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +9 -0
- package/dist/errors/index.js +18 -1
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +84 -32
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -5
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +7 -5
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
- package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
- package/dist/model-fit/floor.d.ts +31 -0
- package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
- package/dist/model-fit/kv-cache.d.ts +23 -0
- package/dist/model-fit/kv-cache.js +85 -0
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
- package/dist/model-fit/platform.d.ts +14 -0
- package/dist/model-fit/platform.js +26 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
- package/dist/model-fit/request/create.js +61 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/model-fit/request/engines/audiogen.js +63 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
- package/dist/model-fit/request/engines/llama.js +44 -0
- package/dist/model-fit/request/engines/nmt.d.ts +8 -0
- package/dist/model-fit/request/engines/nmt.js +11 -0
- package/dist/model-fit/request/engines/ocr.d.ts +9 -0
- package/dist/model-fit/request/engines/ocr.js +13 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1072 -815
- package/dist/models/registry/models.js +3704 -3405
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +406 -3
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
- package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
- package/dist/plugins/builtin/index.d.ts +15 -12
- package/dist/plugins/builtin/index.js +15 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
- package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
- package/dist/plugins/ops/kv-cache-utils.js +25 -33
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +33 -19
- package/dist/plugins/ops/transcribe.js +18 -2
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +3 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +9 -1
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +22 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +7 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +106 -43
- package/dist/schemas/assess-model-fit.js +69 -32
- package/dist/schemas/audio-gen.d.ts +199 -19
- package/dist/schemas/audio-gen.js +183 -51
- package/dist/schemas/batch-completion-stream.d.ts +210 -36
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +3 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +24 -3
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +1465 -162
- package/dist/schemas/common.js +37 -31
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +243 -39
- package/dist/schemas/completion-stream.js +100 -29
- package/dist/schemas/config.d.ts +1326 -18
- package/dist/schemas/config.js +30 -8
- package/dist/schemas/decision.d.ts +610 -0
- package/dist/schemas/decision.js +219 -0
- package/dist/schemas/download-asset.d.ts +10 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +9 -1
- package/dist/schemas/engine-addon-map.js +26 -5
- package/dist/schemas/errors.d.ts +2 -0
- package/dist/schemas/errors.js +11 -1
- package/dist/schemas/index.d.ts +51 -48
- package/dist/schemas/index.js +50 -47
- package/dist/schemas/llamacpp-config.d.ts +222 -6
- package/dist/schemas/llamacpp-config.js +129 -7
- package/dist/schemas/load-model.d.ts +7960 -1492
- package/dist/schemas/load-model.js +122 -10
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +428 -9
- package/dist/schemas/model-config-schemas.js +16 -11
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +4 -3
- package/dist/schemas/model-resource-profile.d.ts +4 -4
- package/dist/schemas/model-resource-profile.js +2 -5
- package/dist/schemas/model-src-utils.d.ts +14 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +14 -1
- package/dist/schemas/model-types.js +4 -0
- package/dist/schemas/ocr.d.ts +76 -0
- package/dist/schemas/ocr.js +43 -1
- package/dist/schemas/plugin.d.ts +16 -4
- package/dist/schemas/plugin.js +12 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +34 -0
- package/dist/schemas/registry.js +5 -1
- package/dist/schemas/reload-config.d.ts +14 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +82 -0
- package/dist/schemas/sdcpp-config.js +41 -22
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +463 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +20 -0
- package/dist/schemas/transcription-config.js +54 -4
- package/dist/schemas/transcription.d.ts +47 -5
- package/dist/schemas/transcription.js +52 -5
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +40 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +3 -2
- package/dist/surface.js +3 -2
- package/dist/utils/addon-responses.d.ts +3 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +13 -9
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +2 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +68 -51
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -28
- package/dist/resources/model-fit/types.d.ts +0 -120
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
|
@@ -1,15 +1,6 @@
|
|
|
1
|
-
import { getModelResourceProfile } from '
|
|
2
|
-
import {
|
|
3
|
-
import { estimateLlm } from './estimators/llm.js';
|
|
4
|
-
import { estimateWhisper } from './estimators/whisper.js';
|
|
5
|
-
import { computeFloor, FLOOR_VERSION } from './floor.js';
|
|
1
|
+
import { getModelResourceProfile } from '../models/registry/resource-profiles.js';
|
|
2
|
+
import { computeFloor, FLOOR_VERSION } from '../model-fit/floor.js';
|
|
6
3
|
const GIB = 1024 * 1024 * 1024;
|
|
7
|
-
/** Engines phase 1 can estimate, and which estimator owns each. */
|
|
8
|
-
const ESTIMATORS = {
|
|
9
|
-
'llamacpp-completion': estimateLlm,
|
|
10
|
-
'llamacpp-embedding': estimateLlm,
|
|
11
|
-
'whispercpp-transcription': estimateWhisper
|
|
12
|
-
};
|
|
13
4
|
const MOBILE_PLATFORMS = ['android-arm64', 'ios-arm64'];
|
|
14
5
|
// Every allocation, GPU included, is system RAM: the system budget always bounds the load.
|
|
15
6
|
const UNIFIED_MEMORY_PLATFORMS = [
|
|
@@ -18,44 +9,53 @@ const UNIFIED_MEMORY_PLATFORMS = [
|
|
|
18
9
|
'ios-arm64'
|
|
19
10
|
];
|
|
20
11
|
/**
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* Apple silicon and mobile calibrate on their GPU already.
|
|
12
|
+
* What one fitter answered: its measurement where the figures divide, its bare
|
|
13
|
+
* verdict where they do not, and `undefined` where it reached no verdict.
|
|
24
14
|
*/
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
'
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
15
|
+
function nativeEvaluation(native) {
|
|
16
|
+
const fit = native?.fit;
|
|
17
|
+
if (!fit || fit.verdict === 'unknown')
|
|
18
|
+
return undefined;
|
|
19
|
+
const reasons = fit.message === undefined ? [fit.reason] : [fit.reason, fit.message];
|
|
20
|
+
const bare = {
|
|
21
|
+
kind: 'bare',
|
|
22
|
+
verdict: fit.verdict,
|
|
23
|
+
estimatorVersion: fit.estimatorVersion,
|
|
24
|
+
reasons
|
|
25
|
+
};
|
|
26
|
+
const projection = fit.projection;
|
|
27
|
+
if (!projection)
|
|
28
|
+
return bare;
|
|
29
|
+
const { weightsBytes, contextBytes, computeBytes, hostBytes } = projection;
|
|
30
|
+
if (weightsBytes === undefined || contextBytes === undefined || computeBytes === undefined) {
|
|
31
|
+
return bare;
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
kind: 'native',
|
|
35
|
+
persistent: weightsBytes + contextBytes,
|
|
36
|
+
working: computeBytes,
|
|
37
|
+
hostBytes: hostBytes ?? 0,
|
|
38
|
+
estimatorVersion: fit.estimatorVersion,
|
|
39
|
+
verdict: fit.verdict,
|
|
40
|
+
reasons
|
|
41
|
+
};
|
|
42
|
+
}
|
|
31
43
|
/**
|
|
32
44
|
* Turns catalog profiles plus a fresh system-memory sample into an advisory
|
|
33
45
|
* verdict.
|
|
34
46
|
*
|
|
35
47
|
* Pure: every input is passed in, so the same call is testable without a worker,
|
|
36
|
-
* a device, or a network. Sampling, platform detection and
|
|
37
|
-
* happen in the handler.
|
|
48
|
+
* a device, or a network. Sampling, platform detection and the engine fitter
|
|
49
|
+
* calls happen in the handler.
|
|
38
50
|
*/
|
|
39
51
|
export function assessModelFitFromResources(options) {
|
|
40
|
-
const { models, execution, resources, platform
|
|
52
|
+
const { models, execution, resources, platform } = options;
|
|
41
53
|
const resolveProfile = options.resolveProfile ?? getModelResourceProfile;
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
// Where the model would execute decides which evidence can bound it, and
|
|
45
|
-
// which fixture describes the load. See METHODOLOGY.md, "Which fixture a
|
|
46
|
-
// host gets".
|
|
47
|
-
const mustPlaceGpu = platform !== undefined && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources);
|
|
48
|
-
const placement = mustPlaceGpu && platform ? resolveGpuPlacement(resources, platform) : undefined;
|
|
49
|
-
const gpuCalibration = placement && platform
|
|
50
|
-
? placement.kind === 'device'
|
|
51
|
-
? resolveGpuCalibration(platform, placement.backend)
|
|
52
|
-
: resolveSharedGpuCalibration(platform, placement.backend)
|
|
54
|
+
const placement = platform !== undefined && hasGpu(resources)
|
|
55
|
+
? resolveGpuPlacement(resources, platform)
|
|
53
56
|
: undefined;
|
|
54
|
-
|
|
55
|
-
const
|
|
56
|
-
const onIntegrated = placement?.kind === 'shared' && gpuCalibration ? placement : undefined;
|
|
57
|
-
const gpuMode = onDevice !== undefined || onIntegrated !== undefined;
|
|
58
|
-
const effectiveCalibration = gpuMode ? gpuCalibration : calibration;
|
|
57
|
+
const onDevice = placement?.kind === 'device' ? placement : undefined;
|
|
58
|
+
const onIntegrated = placement?.kind === 'shared' ? placement : undefined;
|
|
59
59
|
const basis = onDevice
|
|
60
60
|
? onDevice.targets[0].scope === 'budget'
|
|
61
61
|
? 'device-budget'
|
|
@@ -87,106 +87,86 @@ export function assessModelFitFromResources(options) {
|
|
|
87
87
|
// A discrete-GPU load is paid for in system RAM too. In shared mode the
|
|
88
88
|
// system budget already *is* the budget.
|
|
89
89
|
const alsoBoundBy = onDevice ? resolveBudget(resources, platform, 'system-memory', []) : undefined;
|
|
90
|
+
// The computed floor says something only where the model executes out of the
|
|
91
|
+
// memory the budget measures. A discrete card holds the weights in its own
|
|
92
|
+
// memory, so the system budget bounds nothing there.
|
|
93
|
+
const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
|
|
94
|
+
const evaluated = models.map((candidate, index) => evaluate(candidate, resolveProfile, floorApplies, options.nativeFits?.[index], models.length === 1));
|
|
95
|
+
const results = evaluated.map(({ result }) => result);
|
|
90
96
|
if (!platform) {
|
|
91
97
|
reasons.push('the runtime platform is not one this assessment covers');
|
|
92
98
|
}
|
|
93
|
-
else if (mustPlaceGpu && !gpuMode) {
|
|
94
|
-
// Name the missing fixture: the platform's own coefficients exist here,
|
|
95
|
-
// they are just the wrong ones.
|
|
96
|
-
reasons.push(!placement
|
|
97
|
-
? 'a GPU is reported but its readings cannot say where the model would execute, so no estimate can be defended'
|
|
98
|
-
: placement.kind === 'device'
|
|
99
|
-
? `no validated calibration for ${platform} on ${placement.backend}, so no estimate can be defended`
|
|
100
|
-
: `no validated calibration for ${platform} on an integrated ${placement.backend} GPU, so no estimate can be defended`);
|
|
101
|
-
}
|
|
102
|
-
else if (!effectiveCalibration) {
|
|
103
|
-
reasons.push(`no validated calibration for ${platform}, so no estimate can be defended`);
|
|
104
|
-
}
|
|
105
|
-
else if (effectiveCalibration.measuredAt) {
|
|
106
|
-
assumptions.push(calibrationAssumption(platform, effectiveCalibration));
|
|
107
|
-
}
|
|
108
|
-
// Without coefficients, the computed floor still says something — but only
|
|
109
|
-
// where the model executes out of the memory the budget measures. A discrete
|
|
110
|
-
// card holds the weights in its own memory, so the system budget bounds
|
|
111
|
-
// nothing there and the floor has no budget to be compared against.
|
|
112
|
-
const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
|
|
113
|
-
const evaluated = models.map((candidate) => evaluate(candidate, platform, effectiveCalibration, resources, resolveProfile, gpuMode, floorApplies));
|
|
114
99
|
for (const { result } of evaluated) {
|
|
100
|
+
if (result.kind === 'bare')
|
|
101
|
+
continue;
|
|
115
102
|
for (const assumption of result.assumptions ?? []) {
|
|
116
103
|
if (!assumptions.includes(assumption))
|
|
117
104
|
assumptions.push(assumption);
|
|
118
105
|
}
|
|
119
106
|
}
|
|
120
107
|
const modelResults = evaluated.map(({ candidate, result }) => toModelResult(candidate, result, budget, deviceBudgets, alsoBoundBy));
|
|
121
|
-
const
|
|
122
|
-
const estimates = results.filter((result) => {
|
|
123
|
-
return result.kind === 'estimate';
|
|
124
|
-
});
|
|
108
|
+
const natives = results.filter((result) => result.kind === 'native');
|
|
125
109
|
const anyUnknown = results.some((result) => result.kind === 'unknown');
|
|
126
110
|
const anyFloor = results.some((result) => result.kind === 'floor');
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
//
|
|
130
|
-
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
111
|
+
const bares = results.filter((result) => result.kind === 'bare');
|
|
112
|
+
const allNative = natives.length === results.length && natives.length > 0;
|
|
113
|
+
// Measured bytes are exact, so a set of them can confirm a fit. Mixed with a
|
|
114
|
+
// floor the total is a lower bound again, and `combinedFloor` carries it.
|
|
115
|
+
const combinedNative = allNative ? aggregateNative(natives, execution) : undefined;
|
|
116
|
+
const combinedFloor = anyUnknown || bares.length > 0 ? undefined : aggregateFloor(results, execution);
|
|
117
|
+
// A bare verdict is the engine's own, so it names the evidence; otherwise the
|
|
118
|
+
// weakest candidate does. Absent when a candidate rests on nothing, so an
|
|
119
|
+
// `unknown` carrying `evidence` is a near-miss rather than a missing model.
|
|
120
|
+
const evidence = bares.length > 0
|
|
121
|
+
? 'native-fit'
|
|
122
|
+
: anyUnknown
|
|
123
|
+
? undefined
|
|
124
|
+
: anyFloor
|
|
125
|
+
? 'computed-only'
|
|
126
|
+
: allNative
|
|
127
|
+
? 'native-fit'
|
|
128
|
+
: undefined;
|
|
142
129
|
if (anyUnknown) {
|
|
143
|
-
reasons.push('at least one model could not be
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
130
|
+
reasons.push('at least one model could not be assessed, so the combined verdict is unknown');
|
|
131
|
+
}
|
|
132
|
+
// An engine refuses for reasons its own figures do not carry — no device
|
|
133
|
+
// could hold the placement, a load shape it will not run — so its refusal
|
|
134
|
+
// stands whatever the arithmetic says.
|
|
135
|
+
const refused = bares.some((bare) => bare.verdict === 'does-not-fit') ||
|
|
136
|
+
natives.some((native) => native.verdict === 'does-not-fit');
|
|
137
|
+
// The engine measured this model against this device, so its verdict stands
|
|
138
|
+
// where the arithmetic cannot run. A set still needs a budget, since each
|
|
139
|
+
// model was measured alone.
|
|
140
|
+
const engineAnswered = bares.length > 0 || (!budget && models.length === 1 && natives.length === 1);
|
|
141
|
+
const verdict = refused
|
|
142
|
+
? 'likely-too-large'
|
|
143
|
+
: engineAnswered
|
|
144
|
+
? 'likely-fits'
|
|
145
|
+
: !budget
|
|
146
|
+
? 'unknown'
|
|
147
|
+
: combinedNative !== undefined
|
|
148
|
+
? verdictAgainst(combinedNative, deviceBudgets ?? [budget], alsoBoundBy)
|
|
149
|
+
: combinedFloor !== undefined
|
|
150
|
+
? floorVerdict(combinedFloor, deviceBudgets ?? [budget], alsoBoundBy)
|
|
151
|
+
: 'unknown';
|
|
152
|
+
if (onDevice && combinedNative !== undefined && alsoBoundBy) {
|
|
153
153
|
reasons.push('a GPU load is also paid for in system RAM, so every verdict is the more pessimistic of the GPU and system budgets');
|
|
154
154
|
}
|
|
155
|
-
if (deviceBudgets && deviceBudgets.length > 1 &&
|
|
155
|
+
if (deviceBudgets && deviceBudgets.length > 1 && combinedNative !== undefined) {
|
|
156
156
|
reasons.push('more than one usable GPU is reported, so a fit has to hold on the smallest of them and a refusal on the largest');
|
|
157
157
|
}
|
|
158
|
-
if (budget &&
|
|
158
|
+
if (budget && (combinedNative !== undefined || combinedFloor !== undefined)) {
|
|
159
159
|
reasons.push(execution === 'concurrent'
|
|
160
160
|
? 'all models counted as resident with every working peak added'
|
|
161
161
|
: 'all models counted as resident with only the largest working peak added');
|
|
162
162
|
}
|
|
163
|
-
else if (budget && combinedFloor !== undefined) {
|
|
164
|
-
reasons.push(execution === 'concurrent'
|
|
165
|
-
? 'all models counted as resident with their floors summed and every calibrated working peak added'
|
|
166
|
-
: 'all models counted as resident with their floors summed and only the largest calibrated working peak added');
|
|
167
|
-
}
|
|
168
163
|
if (anyFloor) {
|
|
169
164
|
reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
|
|
170
165
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
// what the coefficients above approximate. The estimator's reasons and
|
|
174
|
-
// assumptions describe the path not taken, so none of them are reported.
|
|
175
|
-
const native = nativeModelResult(options.nativeFit, modelResults);
|
|
176
|
-
if (native) {
|
|
177
|
-
return {
|
|
178
|
-
verdict: native.verdict,
|
|
179
|
-
basis,
|
|
180
|
-
execution,
|
|
181
|
-
evidence: 'native-fit',
|
|
182
|
-
...(budget && { budget }),
|
|
183
|
-
models: [native],
|
|
184
|
-
reasons: ['the engine fitter read the registry description of this model'],
|
|
185
|
-
assumptions: []
|
|
186
|
-
};
|
|
166
|
+
if (allNative) {
|
|
167
|
+
reasons.push('every model was measured by the engine that would run it');
|
|
187
168
|
}
|
|
188
|
-
const declined
|
|
189
|
-
if (declined !== undefined) {
|
|
169
|
+
for (const declined of nativeFitDeclines(options.nativeFits, models)) {
|
|
190
170
|
reasons.push(declined);
|
|
191
171
|
}
|
|
192
172
|
return {
|
|
@@ -195,54 +175,31 @@ export function assessModelFitFromResources(options) {
|
|
|
195
175
|
execution,
|
|
196
176
|
...(evidence && { evidence }),
|
|
197
177
|
...(budget && { budget }),
|
|
198
|
-
...(
|
|
199
|
-
estimate: { lowerBoundBytes: combined.lower, upperBoundBytes: combined.upper }
|
|
200
|
-
}),
|
|
201
|
-
...(anyFloor && combinedFloor !== undefined && { floorBytes: combinedFloor }),
|
|
178
|
+
...(anyFloor && combinedFloor !== undefined && { floorBytes: totalBytes(combinedFloor) }),
|
|
202
179
|
models: modelResults,
|
|
203
180
|
reasons,
|
|
204
181
|
assumptions
|
|
205
182
|
};
|
|
206
183
|
}
|
|
207
184
|
/**
|
|
208
|
-
* Why no engine verdict
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
*/
|
|
212
|
-
function nativeFitDecline(nativeFit, unavailable) {
|
|
213
|
-
if (unavailable !== undefined)
|
|
214
|
-
return `no engine fit: ${unavailable}`;
|
|
215
|
-
if (nativeFit?.verdict !== 'unknown')
|
|
216
|
-
return undefined;
|
|
217
|
-
const detail = nativeFit.message === undefined ? nativeFit.reason : nativeFit.message;
|
|
218
|
-
return `the engine fitter reached no verdict: ${detail}`;
|
|
219
|
-
}
|
|
220
|
-
/**
|
|
221
|
-
* The per-model result a native fit produces, or `undefined` when the modelled
|
|
222
|
-
* one stands.
|
|
223
|
-
*
|
|
224
|
-
* The probe answers about one model against the whole machine. Two such answers
|
|
225
|
-
* carry no byte demand that could be summed under one budget, so a multi-model
|
|
226
|
-
* assessment keeps the estimator that can aggregate. `unknown` — the probe
|
|
227
|
-
* disabled, the load shape unsupported, the child unusable — is not a verdict
|
|
228
|
-
* and falls through the same way.
|
|
185
|
+
* Why a candidate carries no engine verdict: the fitter never ran for it, or it
|
|
186
|
+
* ran and declined. Each is named with its model, since a set can lose one
|
|
187
|
+
* model to the fitter and keep the rest.
|
|
229
188
|
*/
|
|
230
|
-
function
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
reasons: nativeFit.message === undefined ? [nativeFit.reason] : [nativeFit.reason, nativeFit.message]
|
|
245
|
-
};
|
|
189
|
+
function nativeFitDeclines(nativeFits, models) {
|
|
190
|
+
const declines = [];
|
|
191
|
+
nativeFits?.forEach((native, index) => {
|
|
192
|
+
const name = models[index]?.model.name ?? `model ${index + 1}`;
|
|
193
|
+
if (native.unavailable !== undefined) {
|
|
194
|
+
declines.push(`no engine fit for ${name}: ${native.unavailable}`);
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
if (native.fit?.verdict !== 'unknown')
|
|
198
|
+
return;
|
|
199
|
+
const detail = native.fit.message === undefined ? native.fit.reason : native.fit.message;
|
|
200
|
+
declines.push(`the engine fitter reached no verdict for ${name}: ${detail}`);
|
|
201
|
+
});
|
|
202
|
+
return declines;
|
|
246
203
|
}
|
|
247
204
|
/**
|
|
248
205
|
* Whether every allocation a load makes comes out of the memory the system (or
|
|
@@ -266,13 +223,16 @@ export function boundBySystemMemory(resources, platform) {
|
|
|
266
223
|
return gpus.every(allocatesFromSystemMemory);
|
|
267
224
|
}
|
|
268
225
|
/**
|
|
269
|
-
* One candidate's evaluation: the
|
|
270
|
-
*
|
|
271
|
-
*
|
|
272
|
-
*
|
|
273
|
-
* a floor.
|
|
226
|
+
* One candidate's evaluation: the engine fitter's measurement when it ran,
|
|
227
|
+
* otherwise the computed floor where a floor can be compared at all, otherwise
|
|
228
|
+
* `unknown`. Whatever kept the measurement from forming stays in the reasons,
|
|
229
|
+
* so a floor verdict still says why it is only a floor.
|
|
274
230
|
*/
|
|
275
|
-
function evaluate(candidate,
|
|
231
|
+
function evaluate(candidate, resolveProfile, floorApplies, native, sole) {
|
|
232
|
+
const measured = nativeEvaluation(native);
|
|
233
|
+
if (measured && (measured.kind !== 'bare' || measured.verdict === 'does-not-fit' || sole)) {
|
|
234
|
+
return { candidate, result: measured };
|
|
235
|
+
}
|
|
276
236
|
const profile = resolveProfile(candidate.model.sha256Checksum);
|
|
277
237
|
if (!profile) {
|
|
278
238
|
return { candidate, result: unknown('no resource profile in the catalog for this checksum') };
|
|
@@ -284,46 +244,25 @@ function evaluate(candidate, platform, calibration, resources, resolveProfile, g
|
|
|
284
244
|
result: unknown('an entry in `artifacts` has no resource profile in the catalog')
|
|
285
245
|
};
|
|
286
246
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
247
|
+
if (!floorApplies) {
|
|
248
|
+
return {
|
|
249
|
+
candidate,
|
|
250
|
+
result: unknown('no engine verdict, and the floor has no budget to be compared against')
|
|
251
|
+
};
|
|
252
|
+
}
|
|
290
253
|
const floor = computeFloor({ profile, workload: candidate.workload, extraArtifactBytes: extra });
|
|
291
254
|
return {
|
|
292
255
|
candidate,
|
|
293
256
|
result: {
|
|
294
257
|
kind: 'floor',
|
|
295
258
|
bytes: floor.bytes,
|
|
296
|
-
reasons:
|
|
297
|
-
assumptions:
|
|
259
|
+
reasons: floor.reasons,
|
|
260
|
+
assumptions: floor.assumptions
|
|
298
261
|
}
|
|
299
262
|
};
|
|
300
263
|
}
|
|
301
264
|
function unknown(reason) {
|
|
302
|
-
return { kind: 'unknown',
|
|
303
|
-
}
|
|
304
|
-
/** Runs the calibrated estimator for a candidate, or says why it cannot. */
|
|
305
|
-
function estimate(candidate, profile, extraArtifactBytes, platform, calibration, resources, gpuMode) {
|
|
306
|
-
if (!calibration) {
|
|
307
|
-
return unknown(platform
|
|
308
|
-
? `no validated calibration for ${platform}`
|
|
309
|
-
: 'the runtime platform is not one this assessment covers');
|
|
310
|
-
}
|
|
311
|
-
// A GPU is present and neither GPU mode engaged, so the only coefficients
|
|
312
|
-
// left were measured with the offload disabled — not how this host runs.
|
|
313
|
-
if (!gpuMode && platform && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources)) {
|
|
314
|
-
return unknown('a GPU is present, so the model executes on it rather than on the CPU this platform’s coefficients were measured against');
|
|
315
|
-
}
|
|
316
|
-
const estimator = ESTIMATORS[profile.engine];
|
|
317
|
-
if (!estimator) {
|
|
318
|
-
return unknown(`engine '${profile.engine}' has no estimator in this phase`);
|
|
319
|
-
}
|
|
320
|
-
return estimator({
|
|
321
|
-
profile,
|
|
322
|
-
workload: candidate.workload,
|
|
323
|
-
extraArtifactBytes,
|
|
324
|
-
calibration,
|
|
325
|
-
hasGpu: hasGpu(resources)
|
|
326
|
-
});
|
|
265
|
+
return { kind: 'unknown', reasons: [reason] };
|
|
327
266
|
}
|
|
328
267
|
/**
|
|
329
268
|
* Sums the artifact bytes of companion constants.
|
|
@@ -343,24 +282,6 @@ function extraArtifactBytes(candidate, resolveProfile) {
|
|
|
343
282
|
}
|
|
344
283
|
return total;
|
|
345
284
|
}
|
|
346
|
-
/**
|
|
347
|
-
* States when, and under what conditions, this platform's coefficients were
|
|
348
|
-
* measured.
|
|
349
|
-
*
|
|
350
|
-
* The backend belongs in the result because the buffers these coefficients
|
|
351
|
-
* cover are allocated by it, while the coefficients themselves are keyed by
|
|
352
|
-
* platform alone — so a fixture measured on one backend is being applied to
|
|
353
|
-
* every backend on that platform. That is a real caveat, and the caller is
|
|
354
|
-
* entitled to see it rather than read the fixture.
|
|
355
|
-
*/
|
|
356
|
-
function calibrationAssumption(platform, calibration) {
|
|
357
|
-
const base = `${platform} coefficients were calibrated on ${calibration.measuredAt}`;
|
|
358
|
-
const on = calibration.measuredOn;
|
|
359
|
-
if (!on)
|
|
360
|
-
return base;
|
|
361
|
-
const device = on.device ? ` (${on.device})` : '';
|
|
362
|
-
return `${base} against a ${on.backend} backend${device}, at ${on.kvElementBytes} bytes per KV element`;
|
|
363
|
-
}
|
|
364
285
|
/**
|
|
365
286
|
* The GPUs the engine could actually execute on: `chooseBackend` passes over a
|
|
366
287
|
* paravirtual adapter and a device with no graphics API this build talks to,
|
|
@@ -388,8 +309,8 @@ function tooSmallToHostAModel(gpu) {
|
|
|
388
309
|
return (gpu.memoryTotalBytes.status === 'supported' && gpu.memoryTotalBytes.value < MIN_USABLE_GPU_BYTES);
|
|
389
310
|
}
|
|
390
311
|
// `gpuType.VIRTUAL`: the virtio / VMware / Hyper-V adapter a VM exposes. It
|
|
391
|
-
// has no compute backend, so counting it as a GPU would
|
|
392
|
-
//
|
|
312
|
+
// has no compute backend, so counting it as a GPU would budget every VM
|
|
313
|
+
// against memory nothing can allocate.
|
|
393
314
|
const GPU_TYPE_VIRTUAL = 3;
|
|
394
315
|
function isVirtualDisplayAdapter(gpu) {
|
|
395
316
|
return gpu.type.status === 'supported' && gpu.type.value === GPU_TYPE_VIRTUAL;
|
|
@@ -410,7 +331,7 @@ function allocatesFromSystemMemory(gpu) {
|
|
|
410
331
|
* APU also exposes for its carve-out — a Ryzen 5000U reported `dedicated` with
|
|
411
332
|
* over a gigabyte of "VRAM". Vulkan calls the same device INTEGRATED_GPU, but
|
|
412
333
|
* that is not in the collector, so an AMD GPU on linux cannot be placed and
|
|
413
|
-
* assesses as `unknown
|
|
334
|
+
* assesses as `unknown`.
|
|
414
335
|
*/
|
|
415
336
|
function integratedIsIndistinguishable(gpu, platform) {
|
|
416
337
|
if (!platform.startsWith('linux'))
|
|
@@ -418,8 +339,8 @@ function integratedIsIndistinguishable(gpu, platform) {
|
|
|
418
339
|
return gpu.driverName.status === 'supported' && gpu.driverName.value === 'amdgpu';
|
|
419
340
|
}
|
|
420
341
|
// Ordered by the backends the addon actually builds, not by what the device's
|
|
421
|
-
// drivers advertise:
|
|
422
|
-
//
|
|
342
|
+
// drivers advertise: an NVIDIA host advertises both CUDA and Vulkan, and every
|
|
343
|
+
// load on it reports `ggml_vulkan`, never `ggml_cuda`. The
|
|
423
344
|
// engine's own choice is not observable from here (`chooseBackend` is C++ and
|
|
424
345
|
// only reaches the llama log), so this order has to track the addon's build.
|
|
425
346
|
const GPU_BACKENDS = ['metal', 'vulkan', 'rocm', 'cuda', 'levelZero', 'opencl'];
|
|
@@ -451,9 +372,8 @@ function resolveGpuPlacement(resources, platform) {
|
|
|
451
372
|
if (deviceBacked.length === 0) {
|
|
452
373
|
const shared = gpus[0];
|
|
453
374
|
const backend = backendOf(shared);
|
|
454
|
-
//
|
|
455
|
-
//
|
|
456
|
-
// set of coefficients, no estimate. They share the budget either way.
|
|
375
|
+
// The engine picks one of them, so the same rule applies as for cards: no
|
|
376
|
+
// single backend, no placement. They share the budget either way.
|
|
457
377
|
if (gpus.some((gpu) => backendOf(gpu) !== backend))
|
|
458
378
|
return undefined;
|
|
459
379
|
return {
|
|
@@ -489,8 +409,8 @@ function resolveGpuPlacement(resources, platform) {
|
|
|
489
409
|
...(gpu.name.status === 'supported' && { device: gpu.name.value })
|
|
490
410
|
});
|
|
491
411
|
}
|
|
492
|
-
//
|
|
493
|
-
// either
|
|
412
|
+
// A placement names one backend and one basis, so cards that disagree on
|
|
413
|
+
// either name no device to budget against.
|
|
494
414
|
const backend = targets[0].backend;
|
|
495
415
|
if (targets.some((target) => target.backend !== backend))
|
|
496
416
|
return undefined;
|
|
@@ -513,22 +433,31 @@ function gpuBudget(target, platform) {
|
|
|
513
433
|
function tightest(budgets) {
|
|
514
434
|
return budgets.reduce((least, budget) => budget.availableAfterReserveBytes < least.availableAfterReserveBytes ? budget : least);
|
|
515
435
|
}
|
|
436
|
+
/**
|
|
437
|
+
* What the candidate budgets have to hold. `alsoBoundBy` is present only where
|
|
438
|
+
* they are a card's own memory, which host bytes never come out of; without it
|
|
439
|
+
* they are system memory and everything lands there.
|
|
440
|
+
*/
|
|
441
|
+
function chargedToDevice(demand, alsoBoundBy) {
|
|
442
|
+
return alsoBoundBy ? demand.deviceBytes : totalBytes(demand);
|
|
443
|
+
}
|
|
516
444
|
/**
|
|
517
445
|
* `candidates` are alternatives — the engine pins the model to one of them, and
|
|
518
446
|
* which one is not observable here — so a fit has to hold on the smallest and a
|
|
519
447
|
* refusal on the largest. `alsoBoundBy` is a conjunction: a GPU load is paid
|
|
520
448
|
* for in system RAM too, so both bounds apply.
|
|
521
449
|
*/
|
|
522
|
-
function verdictAgainst(
|
|
450
|
+
function verdictAgainst(demand, candidates, alsoBoundBy) {
|
|
523
451
|
const room = candidates.map((budget) => budget.availableAfterReserveBytes);
|
|
524
|
-
const
|
|
452
|
+
const onDevice = chargedToDevice(demand, alsoBoundBy);
|
|
453
|
+
const primary = onDevice <= Math.min(...room)
|
|
525
454
|
? 'likely-fits'
|
|
526
|
-
:
|
|
455
|
+
: onDevice > Math.max(...room)
|
|
527
456
|
? 'likely-too-large'
|
|
528
457
|
: 'unknown';
|
|
529
458
|
if (!alsoBoundBy)
|
|
530
459
|
return primary;
|
|
531
|
-
return worst(primary,
|
|
460
|
+
return worst(primary, totalBytes(demand) > alsoBoundBy.availableAfterReserveBytes ? 'likely-too-large' : 'likely-fits');
|
|
532
461
|
}
|
|
533
462
|
/** The more pessimistic of two verdicts. */
|
|
534
463
|
function worst(a, b) {
|
|
@@ -629,31 +558,6 @@ function reserveBytes(availableBytes, platform) {
|
|
|
629
558
|
const mobile = platform !== undefined && MOBILE_PLATFORMS.includes(platform);
|
|
630
559
|
return Math.min(mobile ? 1 * GIB : 2 * GIB, Math.floor(availableBytes * 0.2));
|
|
631
560
|
}
|
|
632
|
-
/**
|
|
633
|
-
* Combines per-model bounds under the declared execution mode.
|
|
634
|
-
*
|
|
635
|
-
* Every model is resident either way — what differs is the working peak:
|
|
636
|
-
* `sequential` counts only the largest, `concurrent` counts them all.
|
|
637
|
-
*/
|
|
638
|
-
function aggregate(estimates, execution) {
|
|
639
|
-
let persistentLower = 0;
|
|
640
|
-
let persistentUpper = 0;
|
|
641
|
-
let workingLower = 0;
|
|
642
|
-
let workingUpper = 0;
|
|
643
|
-
for (const estimate of estimates) {
|
|
644
|
-
persistentLower += estimate.persistent.lower;
|
|
645
|
-
persistentUpper += estimate.persistent.upper;
|
|
646
|
-
if (execution === 'concurrent') {
|
|
647
|
-
workingLower += estimate.working.lower;
|
|
648
|
-
workingUpper += estimate.working.upper;
|
|
649
|
-
}
|
|
650
|
-
else {
|
|
651
|
-
workingLower = Math.max(workingLower, estimate.working.lower);
|
|
652
|
-
workingUpper = Math.max(workingUpper, estimate.working.upper);
|
|
653
|
-
}
|
|
654
|
-
}
|
|
655
|
-
return { lower: persistentLower + workingLower, upper: persistentUpper + workingUpper };
|
|
656
|
-
}
|
|
657
561
|
/**
|
|
658
562
|
* The combined lower bound when at least one candidate has only a floor: every
|
|
659
563
|
* estimate contributes its own lower bound, every floor its bytes, aggregated
|
|
@@ -664,40 +568,61 @@ function aggregate(estimates, execution) {
|
|
|
664
568
|
function aggregateFloor(results, execution) {
|
|
665
569
|
let persistent = 0;
|
|
666
570
|
let working = 0;
|
|
571
|
+
let hostBytes = 0;
|
|
667
572
|
for (const result of results) {
|
|
668
|
-
if (result.kind === 'unknown')
|
|
573
|
+
if (result.kind === 'unknown' || result.kind === 'bare')
|
|
669
574
|
return undefined;
|
|
670
575
|
if (result.kind === 'floor') {
|
|
671
576
|
persistent += result.bytes;
|
|
672
577
|
continue;
|
|
673
578
|
}
|
|
674
|
-
persistent += result.persistent
|
|
579
|
+
persistent += result.persistent;
|
|
580
|
+
hostBytes += result.hostBytes;
|
|
581
|
+
working =
|
|
582
|
+
execution === 'concurrent' ? working + result.working : Math.max(working, result.working);
|
|
583
|
+
}
|
|
584
|
+
return { deviceBytes: persistent + working, hostBytes };
|
|
585
|
+
}
|
|
586
|
+
function totalBytes(demand) {
|
|
587
|
+
return demand.deviceBytes + demand.hostBytes;
|
|
588
|
+
}
|
|
589
|
+
/**
|
|
590
|
+
* The combined demand when every candidate was measured by its own engine.
|
|
591
|
+
* Each fitter answered about one model against the whole machine, so only the
|
|
592
|
+
* bytes compose: persistent is always resident, and the working peak is summed
|
|
593
|
+
* under `concurrent` and maximised under `sequential`.
|
|
594
|
+
*/
|
|
595
|
+
function aggregateNative(natives, execution) {
|
|
596
|
+
let persistent = 0;
|
|
597
|
+
let working = 0;
|
|
598
|
+
let hostBytes = 0;
|
|
599
|
+
for (const native of natives) {
|
|
600
|
+
persistent += native.persistent;
|
|
601
|
+
hostBytes += native.hostBytes;
|
|
675
602
|
working =
|
|
676
|
-
execution === 'concurrent'
|
|
677
|
-
? working + result.working.lower
|
|
678
|
-
: Math.max(working, result.working.lower);
|
|
603
|
+
execution === 'concurrent' ? working + native.working : Math.max(working, native.working);
|
|
679
604
|
}
|
|
680
|
-
return persistent + working;
|
|
605
|
+
return { deviceBytes: persistent + working, hostBytes };
|
|
681
606
|
}
|
|
682
607
|
/**
|
|
683
608
|
* A floor can only refuse: over the largest candidate budget it is
|
|
684
609
|
* `likely-too-large`, and anything else is `unknown`, because the cost above
|
|
685
610
|
* the floor is unmeasured and could be anything.
|
|
686
611
|
*/
|
|
687
|
-
function floorVerdict(
|
|
612
|
+
function floorVerdict(floor, candidates, alsoBoundBy) {
|
|
688
613
|
const room = candidates.map((budget) => budget.availableAfterReserveBytes);
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
if (estimate.lower > budget)
|
|
614
|
+
if (chargedToDevice(floor, alsoBoundBy) > Math.max(...room))
|
|
615
|
+
return 'likely-too-large';
|
|
616
|
+
if (alsoBoundBy && totalBytes(floor) > alsoBoundBy.availableAfterReserveBytes) {
|
|
693
617
|
return 'likely-too-large';
|
|
694
|
-
|
|
695
|
-
return 'likely-fits';
|
|
618
|
+
}
|
|
696
619
|
return 'unknown';
|
|
697
620
|
}
|
|
698
621
|
function toModelResult(candidate, result, budget,
|
|
699
622
|
/** Every candidate GPU budget, when the host has more than one. */
|
|
700
|
-
deviceBudgets,
|
|
623
|
+
deviceBudgets,
|
|
624
|
+
/** The system budget a GPU load is also paid for out of. */
|
|
625
|
+
alsoBoundBy) {
|
|
701
626
|
if (result.kind === 'unknown') {
|
|
702
627
|
return {
|
|
703
628
|
name: candidate.model.name,
|
|
@@ -707,7 +632,9 @@ deviceBudgets, alsoBoundBy) {
|
|
|
707
632
|
};
|
|
708
633
|
}
|
|
709
634
|
if (result.kind === 'floor') {
|
|
710
|
-
const verdict = budget
|
|
635
|
+
const verdict = budget
|
|
636
|
+
? floorVerdict({ deviceBytes: result.bytes, hostBytes: 0 }, deviceBudgets ?? [budget], alsoBoundBy)
|
|
637
|
+
: 'unknown';
|
|
711
638
|
return {
|
|
712
639
|
name: candidate.model.name,
|
|
713
640
|
verdict,
|
|
@@ -721,23 +648,32 @@ deviceBudgets, alsoBoundBy) {
|
|
|
721
648
|
? 'no usable system-memory sample, so this model has no verdict'
|
|
722
649
|
: verdict === 'likely-too-large'
|
|
723
650
|
? 'the computed floor alone exceeds the budget'
|
|
724
|
-
: 'the computed floor is within the budget, but
|
|
651
|
+
: 'the computed floor is within the budget, but the cost above it is unmeasured, so a fit cannot be claimed'
|
|
725
652
|
]
|
|
726
653
|
};
|
|
727
654
|
}
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
655
|
+
if (result.kind === 'bare') {
|
|
656
|
+
return {
|
|
657
|
+
name: candidate.model.name,
|
|
658
|
+
verdict: result.verdict === 'fit' ? 'likely-fits' : 'likely-too-large',
|
|
659
|
+
evidence: 'native-fit',
|
|
660
|
+
...(candidate.device !== undefined && { device: candidate.device }),
|
|
661
|
+
estimatorVersion: result.estimatorVersion,
|
|
662
|
+
reasons: [...result.reasons]
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
const bytes = aggregateNative([result], 'sequential');
|
|
666
|
+
const verdict = result.verdict === 'does-not-fit'
|
|
667
|
+
? 'likely-too-large'
|
|
668
|
+
: budget
|
|
669
|
+
? verdictAgainst(bytes, deviceBudgets ?? [budget], alsoBoundBy)
|
|
670
|
+
: 'likely-fits';
|
|
732
671
|
return {
|
|
733
672
|
name: candidate.model.name,
|
|
734
|
-
verdict
|
|
735
|
-
evidence: '
|
|
673
|
+
verdict,
|
|
674
|
+
evidence: 'native-fit',
|
|
736
675
|
...(candidate.device !== undefined && { device: candidate.device }),
|
|
737
|
-
estimate: { lowerBoundBytes: total.lower, upperBoundBytes: total.upper },
|
|
738
676
|
estimatorVersion: result.estimatorVersion,
|
|
739
|
-
reasons:
|
|
740
|
-
? [...result.reasons]
|
|
741
|
-
: [...result.reasons, 'no usable system-memory sample, so this model has no verdict']
|
|
677
|
+
reasons: [...result.reasons]
|
|
742
678
|
};
|
|
743
679
|
}
|