@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
package/dist/model-fit/assess.js
CHANGED
|
@@ -1,679 +1,110 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
*
|
|
14
|
-
|
|
15
|
-
function nativeEvaluation(native) {
|
|
16
|
-
const fit = native?.fit;
|
|
17
|
-
if (!fit || fit.verdict === 'unknown')
|
|
18
|
-
return undefined;
|
|
19
|
-
const reasons = fit.message === undefined ? [fit.reason] : [fit.reason, fit.message];
|
|
20
|
-
const bare = {
|
|
21
|
-
kind: 'bare',
|
|
22
|
-
verdict: fit.verdict,
|
|
23
|
-
estimatorVersion: fit.estimatorVersion,
|
|
24
|
-
reasons
|
|
25
|
-
};
|
|
26
|
-
const projection = fit.projection;
|
|
27
|
-
if (!projection)
|
|
28
|
-
return bare;
|
|
29
|
-
const { weightsBytes, contextBytes, computeBytes, hostBytes } = projection;
|
|
30
|
-
if (weightsBytes === undefined || contextBytes === undefined || computeBytes === undefined) {
|
|
31
|
-
return bare;
|
|
32
|
-
}
|
|
33
|
-
return {
|
|
34
|
-
kind: 'native',
|
|
35
|
-
persistent: weightsBytes + contextBytes,
|
|
36
|
-
working: computeBytes,
|
|
37
|
-
hostBytes: hostBytes ?? 0,
|
|
38
|
-
estimatorVersion: fit.estimatorVersion,
|
|
39
|
-
verdict: fit.verdict,
|
|
40
|
-
reasons
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* Turns catalog profiles plus a fresh system-memory sample into an advisory
|
|
45
|
-
* verdict.
|
|
46
|
-
*
|
|
47
|
-
* Pure: every input is passed in, so the same call is testable without a worker,
|
|
48
|
-
* a device, or a network. Sampling, platform detection and the engine fitter
|
|
49
|
-
* calls happen in the handler.
|
|
50
|
-
*/
|
|
51
|
-
export function assessModelFitFromResources(options) {
|
|
52
|
-
const { models, execution, resources, platform } = options;
|
|
53
|
-
const resolveProfile = options.resolveProfile ?? getModelResourceProfile;
|
|
54
|
-
const placement = platform !== undefined && hasGpu(resources)
|
|
55
|
-
? resolveGpuPlacement(resources, platform)
|
|
56
|
-
: undefined;
|
|
57
|
-
const onDevice = placement?.kind === 'device' ? placement : undefined;
|
|
58
|
-
const onIntegrated = placement?.kind === 'shared' ? placement : undefined;
|
|
59
|
-
const basis = onDevice
|
|
60
|
-
? onDevice.targets[0].scope === 'budget'
|
|
61
|
-
? 'device-budget'
|
|
62
|
-
: 'device-memory'
|
|
63
|
-
: resolveBasis(platform);
|
|
64
|
-
const reasons = [];
|
|
65
|
-
const assumptions = [
|
|
66
|
-
`execution mode '${execution}' is a declared assumption used for aggregation only; the SDK does not schedule, serialize, or reserve anything`,
|
|
67
|
-
`the verdict is advisory and based on ${basisEvidence(basis)} alone; it does not block loadModel and makes no performance claim`
|
|
68
|
-
];
|
|
69
|
-
if (onDevice) {
|
|
70
|
-
const { targets, backend } = onDevice;
|
|
71
|
-
assumptions.push(targets.length === 1
|
|
72
|
-
? `the model is assumed to execute on ${targets[0].device ?? 'the discrete GPU'} via ${backend}, and the budget is that device's own memory`
|
|
73
|
-
: `${targets.length} usable GPUs are reported and the engine pins the model to one of them, an order this side cannot observe; the verdict holds for whichever it picks, and the budget shown is the tightest of them`);
|
|
74
|
-
}
|
|
75
|
-
if (onIntegrated) {
|
|
76
|
-
assumptions.push(`the model is assumed to execute on ${onIntegrated.device ?? 'the integrated GPU'} via ${onIntegrated.backend}; an integrated GPU allocates out of system RAM, so system memory is the budget it draws on`);
|
|
77
|
-
}
|
|
78
|
-
if (platform === 'android-arm64') {
|
|
79
|
-
assumptions.push('android budgets deliberately use system memory: the low-memory killer acts system-wide and native allocations carry no per-process cap like iOS jetsam');
|
|
80
|
-
}
|
|
81
|
-
// Every candidate card carries its own budget; the tightest is reported and
|
|
82
|
-
// the verdict is taken across all of them.
|
|
83
|
-
const deviceBudgets = onDevice?.targets.map((target) => gpuBudget(target, platform));
|
|
84
|
-
const budget = deviceBudgets
|
|
85
|
-
? tightest(deviceBudgets)
|
|
86
|
-
: resolveBudget(resources, platform, basis, reasons);
|
|
87
|
-
// A discrete-GPU load is paid for in system RAM too. In shared mode the
|
|
88
|
-
// system budget already *is* the budget.
|
|
89
|
-
const alsoBoundBy = onDevice ? resolveBudget(resources, platform, 'system-memory', []) : undefined;
|
|
90
|
-
// The computed floor says something only where the model executes out of the
|
|
91
|
-
// memory the budget measures. A discrete card holds the weights in its own
|
|
92
|
-
// memory, so the system budget bounds nothing there.
|
|
93
|
-
const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
|
|
94
|
-
const evaluated = models.map((candidate, index) => evaluate(candidate, resolveProfile, floorApplies, options.nativeFits?.[index], models.length === 1));
|
|
95
|
-
const results = evaluated.map(({ result }) => result);
|
|
96
|
-
if (!platform) {
|
|
97
|
-
reasons.push('the runtime platform is not one this assessment covers');
|
|
98
|
-
}
|
|
99
|
-
for (const { result } of evaluated) {
|
|
100
|
-
if (result.kind === 'bare')
|
|
101
|
-
continue;
|
|
102
|
-
for (const assumption of result.assumptions ?? []) {
|
|
103
|
-
if (!assumptions.includes(assumption))
|
|
104
|
-
assumptions.push(assumption);
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
const modelResults = evaluated.map(({ candidate, result }) => toModelResult(candidate, result, budget, deviceBudgets, alsoBoundBy));
|
|
108
|
-
const natives = results.filter((result) => result.kind === 'native');
|
|
109
|
-
const anyUnknown = results.some((result) => result.kind === 'unknown');
|
|
110
|
-
const anyFloor = results.some((result) => result.kind === 'floor');
|
|
111
|
-
const bares = results.filter((result) => result.kind === 'bare');
|
|
112
|
-
const allNative = natives.length === results.length && natives.length > 0;
|
|
113
|
-
// Measured bytes are exact, so a set of them can confirm a fit. Mixed with a
|
|
114
|
-
// floor the total is a lower bound again, and `combinedFloor` carries it.
|
|
115
|
-
const combinedNative = allNative ? aggregateNative(natives, execution) : undefined;
|
|
116
|
-
const combinedFloor = anyUnknown || bares.length > 0 ? undefined : aggregateFloor(results, execution);
|
|
117
|
-
// A bare verdict is the engine's own, so it names the evidence; otherwise the
|
|
118
|
-
// weakest candidate does. Absent when a candidate rests on nothing, so an
|
|
119
|
-
// `unknown` carrying `evidence` is a near-miss rather than a missing model.
|
|
120
|
-
const evidence = bares.length > 0
|
|
121
|
-
? 'native-fit'
|
|
122
|
-
: anyUnknown
|
|
123
|
-
? undefined
|
|
124
|
-
: anyFloor
|
|
125
|
-
? 'computed-only'
|
|
126
|
-
: allNative
|
|
127
|
-
? 'native-fit'
|
|
128
|
-
: undefined;
|
|
129
|
-
if (anyUnknown) {
|
|
130
|
-
reasons.push('at least one model could not be assessed, so the combined verdict is unknown');
|
|
131
|
-
}
|
|
132
|
-
// An engine refuses for reasons its own figures do not carry — no device
|
|
133
|
-
// could hold the placement, a load shape it will not run — so its refusal
|
|
134
|
-
// stands whatever the arithmetic says.
|
|
135
|
-
const refused = bares.some((bare) => bare.verdict === 'does-not-fit') ||
|
|
136
|
-
natives.some((native) => native.verdict === 'does-not-fit');
|
|
137
|
-
// The engine measured this model against this device, so its verdict stands
|
|
138
|
-
// where the arithmetic cannot run. A set still needs a budget, since each
|
|
139
|
-
// model was measured alone.
|
|
140
|
-
const engineAnswered = bares.length > 0 || (!budget && models.length === 1 && natives.length === 1);
|
|
141
|
-
const verdict = refused
|
|
142
|
-
? 'likely-too-large'
|
|
143
|
-
: engineAnswered
|
|
144
|
-
? 'likely-fits'
|
|
145
|
-
: !budget
|
|
146
|
-
? 'unknown'
|
|
147
|
-
: combinedNative !== undefined
|
|
148
|
-
? verdictAgainst(combinedNative, deviceBudgets ?? [budget], alsoBoundBy)
|
|
149
|
-
: combinedFloor !== undefined
|
|
150
|
-
? floorVerdict(combinedFloor, deviceBudgets ?? [budget], alsoBoundBy)
|
|
151
|
-
: 'unknown';
|
|
152
|
-
if (onDevice && combinedNative !== undefined && alsoBoundBy) {
|
|
153
|
-
reasons.push('a GPU load is also paid for in system RAM, so every verdict is the more pessimistic of the GPU and system budgets');
|
|
154
|
-
}
|
|
155
|
-
if (deviceBudgets && deviceBudgets.length > 1 && combinedNative !== undefined) {
|
|
156
|
-
reasons.push('more than one usable GPU is reported, so a fit has to hold on the smallest of them and a refusal on the largest');
|
|
157
|
-
}
|
|
158
|
-
if (budget && (combinedNative !== undefined || combinedFloor !== undefined)) {
|
|
159
|
-
reasons.push(execution === 'concurrent'
|
|
160
|
-
? 'all models counted as resident with every working peak added'
|
|
161
|
-
: 'all models counted as resident with only the largest working peak added');
|
|
162
|
-
}
|
|
163
|
-
if (anyFloor) {
|
|
164
|
-
reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
|
|
165
|
-
}
|
|
166
|
-
if (allNative) {
|
|
167
|
-
reasons.push('every model was measured by the engine that would run it');
|
|
168
|
-
}
|
|
169
|
-
for (const declined of nativeFitDeclines(options.nativeFits, models)) {
|
|
170
|
-
reasons.push(declined);
|
|
171
|
-
}
|
|
172
|
-
return {
|
|
173
|
-
verdict,
|
|
174
|
-
basis,
|
|
175
|
-
execution,
|
|
176
|
-
...(evidence && { evidence }),
|
|
177
|
-
...(budget && { budget }),
|
|
178
|
-
...(anyFloor && combinedFloor !== undefined && { floorBytes: totalBytes(combinedFloor) }),
|
|
179
|
-
models: modelResults,
|
|
180
|
-
reasons,
|
|
181
|
-
assumptions
|
|
182
|
-
};
|
|
183
|
-
}
|
|
184
|
-
/**
|
|
185
|
-
* Why a candidate carries no engine verdict: the fitter never ran for it, or it
|
|
186
|
-
* ran and declined. Each is named with its model, since a set can lose one
|
|
187
|
-
* model to the fitter and keep the rest.
|
|
188
|
-
*/
|
|
189
|
-
function nativeFitDeclines(nativeFits, models) {
|
|
190
|
-
const declines = [];
|
|
191
|
-
nativeFits?.forEach((native, index) => {
|
|
192
|
-
const name = models[index]?.model.name ?? `model ${index + 1}`;
|
|
193
|
-
if (native.unavailable !== undefined) {
|
|
194
|
-
declines.push(`no engine fit for ${name}: ${native.unavailable}`);
|
|
195
|
-
return;
|
|
196
|
-
}
|
|
197
|
-
if (native.fit?.verdict !== 'unknown')
|
|
198
|
-
return;
|
|
199
|
-
const detail = native.fit.message === undefined ? native.fit.reason : native.fit.message;
|
|
200
|
-
declines.push(`the engine fitter reached no verdict for ${name}: ${detail}`);
|
|
201
|
-
});
|
|
202
|
-
return declines;
|
|
203
|
-
}
|
|
204
|
-
/**
|
|
205
|
-
* Whether every allocation a load makes comes out of the memory the system (or
|
|
206
|
-
* process) budget measures — the precondition for the computed floor to be
|
|
207
|
-
* compared against that budget.
|
|
208
|
-
*
|
|
209
|
-
* True on unified-memory platforms whatever the collector reports, and
|
|
210
|
-
* elsewhere when no usable GPU is present or every usable GPU allocates out of
|
|
211
|
-
* system RAM. A card with its own memory, or a device whose class cannot be
|
|
212
|
-
* told (an AMD APU on linux), fails it: the weights may live where the budget
|
|
213
|
-
* cannot see them.
|
|
214
|
-
*/
|
|
215
|
-
export function boundBySystemMemory(resources, platform) {
|
|
216
|
-
if (platform !== undefined && UNIFIED_MEMORY_PLATFORMS.includes(platform))
|
|
217
|
-
return true;
|
|
218
|
-
const gpus = usableGpus(resources);
|
|
219
|
-
if (gpus.length === 0)
|
|
220
|
-
return true;
|
|
221
|
-
if (platform && gpus.some((gpu) => integratedIsIndistinguishable(gpu, platform)))
|
|
222
|
-
return false;
|
|
223
|
-
return gpus.every(allocatesFromSystemMemory);
|
|
224
|
-
}
|
|
225
|
-
/**
|
|
226
|
-
* One candidate's evaluation: the engine fitter's measurement when it ran,
|
|
227
|
-
* otherwise the computed floor where a floor can be compared at all, otherwise
|
|
228
|
-
* `unknown`. Whatever kept the measurement from forming stays in the reasons,
|
|
229
|
-
* so a floor verdict still says why it is only a floor.
|
|
230
|
-
*/
|
|
231
|
-
function evaluate(candidate, resolveProfile, floorApplies, native, sole) {
|
|
232
|
-
const measured = nativeEvaluation(native);
|
|
233
|
-
if (measured && (measured.kind !== 'bare' || measured.verdict === 'does-not-fit' || sole)) {
|
|
234
|
-
return { candidate, result: measured };
|
|
235
|
-
}
|
|
236
|
-
const profile = resolveProfile(candidate.model.sha256Checksum);
|
|
237
|
-
if (!profile) {
|
|
238
|
-
return { candidate, result: unknown('no resource profile in the catalog for this checksum') };
|
|
239
|
-
}
|
|
240
|
-
const extra = extraArtifactBytes(candidate, resolveProfile);
|
|
241
|
-
if (extra === undefined) {
|
|
242
|
-
return {
|
|
243
|
-
candidate,
|
|
244
|
-
result: unknown('an entry in `artifacts` has no resource profile in the catalog')
|
|
245
|
-
};
|
|
246
|
-
}
|
|
247
|
-
if (!floorApplies) {
|
|
248
|
-
return {
|
|
249
|
-
candidate,
|
|
250
|
-
result: unknown('no engine verdict, and the floor has no budget to be compared against')
|
|
251
|
-
};
|
|
252
|
-
}
|
|
253
|
-
const floor = computeFloor({ profile, workload: candidate.workload, extraArtifactBytes: extra });
|
|
254
|
-
return {
|
|
255
|
-
candidate,
|
|
256
|
-
result: {
|
|
257
|
-
kind: 'floor',
|
|
258
|
-
bytes: floor.bytes,
|
|
259
|
-
reasons: floor.reasons,
|
|
260
|
-
assumptions: floor.assumptions
|
|
261
|
-
}
|
|
262
|
-
};
|
|
263
|
-
}
|
|
264
|
-
function unknown(reason) {
|
|
265
|
-
return { kind: 'unknown', reasons: [reason] };
|
|
266
|
-
}
|
|
267
|
-
/**
|
|
268
|
-
* Sums the artifact bytes of companion constants.
|
|
1
|
+
import path from 'bare-path';
|
|
2
|
+
import { ModelTypeRequiredError, PluginNotFoundError } from '../errors/index.js';
|
|
3
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
4
|
+
import { describeAccounting } from '../model-fit/describe.js';
|
|
5
|
+
import { resolveLoadFromStubs } from '../model-fit/description/resolve-load.js';
|
|
6
|
+
import { fitLoad, unknownFit } from '../model-fit/fit-load.js';
|
|
7
|
+
import { combineFits } from '../model-fit/sum.js';
|
|
8
|
+
import { getConfig } from '../runtime/state.js';
|
|
9
|
+
import { isCanonicalModelType, normalizeModelType } from '../schemas/index.js';
|
|
10
|
+
import { inferModelTypeFromModelSrc } from '../schemas/model-src-utils.js';
|
|
11
|
+
import { detectShardedModel } from '../utils/shard-utils.js';
|
|
12
|
+
/**
|
|
13
|
+
* The engine that would run this load, named outright or read off the source,
|
|
14
|
+
* as `loadModel` resolves it.
|
|
269
15
|
*
|
|
270
|
-
* @
|
|
271
|
-
* an incomplete artifact set must not be silently under-counted.
|
|
272
|
-
*/
|
|
273
|
-
function extraArtifactBytes(candidate, resolveProfile) {
|
|
274
|
-
if (!candidate.artifacts || candidate.artifacts.length === 0)
|
|
275
|
-
return 0;
|
|
276
|
-
let total = 0;
|
|
277
|
-
for (const artifact of candidate.artifacts) {
|
|
278
|
-
const profile = resolveProfile(artifact.sha256Checksum);
|
|
279
|
-
if (!profile)
|
|
280
|
-
return undefined;
|
|
281
|
-
total += profile.artifactBytes;
|
|
282
|
-
}
|
|
283
|
-
return total;
|
|
284
|
-
}
|
|
285
|
-
/**
|
|
286
|
-
* The GPUs the engine could actually execute on: `chooseBackend` passes over a
|
|
287
|
-
* paravirtual adapter and a device with no graphics API this build talks to,
|
|
288
|
-
* and falls back to the CPU. The driver flags are library-presence checks, and
|
|
289
|
-
* ggml's own backend needs the same libraries to load.
|
|
290
|
-
*/
|
|
291
|
-
function usableGpus(resources) {
|
|
292
|
-
const gpus = resources.capabilities.gpus;
|
|
293
|
-
if (gpus.status !== 'supported')
|
|
294
|
-
return [];
|
|
295
|
-
return gpus.value.filter((gpu) => !isVirtualDisplayAdapter(gpu) && backendOf(gpu) !== undefined);
|
|
296
|
-
}
|
|
297
|
-
/** Whether a GPU the engine would use is present. Sets the KV-cache default. */
|
|
298
|
-
function hasGpu(resources) {
|
|
299
|
-
return usableGpus(resources).length > 0;
|
|
300
|
-
}
|
|
301
|
-
// An adapter too small to hold any model we assess is not a candidate for one.
|
|
302
|
-
// Windows classifies the Intel iGPU as dedicated because it declares 128 MiB of
|
|
303
|
-
// its own, which is otherwise indistinguishable from a real card by type.
|
|
304
|
-
const MIN_USABLE_GPU_BYTES = GIB;
|
|
305
|
-
// Only excludes a card when the declared memory says so. An unreported total
|
|
306
|
-
// leaves it counted, so an unknown device makes the choice ambiguous rather
|
|
307
|
-
// than silently disappearing from it.
|
|
308
|
-
function tooSmallToHostAModel(gpu) {
|
|
309
|
-
return (gpu.memoryTotalBytes.status === 'supported' && gpu.memoryTotalBytes.value < MIN_USABLE_GPU_BYTES);
|
|
310
|
-
}
|
|
311
|
-
// `gpuType.VIRTUAL`: the virtio / VMware / Hyper-V adapter a VM exposes. It
|
|
312
|
-
// has no compute backend, so counting it as a GPU would budget every VM
|
|
313
|
-
// against memory nothing can allocate.
|
|
314
|
-
const GPU_TYPE_VIRTUAL = 3;
|
|
315
|
-
function isVirtualDisplayAdapter(gpu) {
|
|
316
|
-
return gpu.type.status === 'supported' && gpu.type.value === GPU_TYPE_VIRTUAL;
|
|
317
|
-
}
|
|
318
|
-
/**
|
|
319
|
-
* Whether this GPU's allocations come out of system RAM. `unifiedMemory` says
|
|
320
|
-
* so directly; the Windows iGPU does not, being typed dedicated for a 128 MiB
|
|
321
|
-
* carve-out, so the usable-memory floor is what identifies it.
|
|
16
|
+
* @throws {ModelTypeRequiredError} When neither names one.
|
|
322
17
|
*/
|
|
323
|
-
function
|
|
324
|
-
if (
|
|
325
|
-
return
|
|
326
|
-
|
|
18
|
+
function modelTypeOf(candidate) {
|
|
19
|
+
if (candidate.modelType !== undefined)
|
|
20
|
+
return normalizeModelType(candidate.modelType);
|
|
21
|
+
const inferred = inferModelTypeFromModelSrc(candidate.modelSrc);
|
|
22
|
+
if (inferred === undefined)
|
|
23
|
+
throw new ModelTypeRequiredError();
|
|
24
|
+
return normalizeModelType(inferred);
|
|
327
25
|
}
|
|
328
26
|
/**
|
|
329
|
-
* Whether the
|
|
330
|
-
*
|
|
331
|
-
* APU also exposes for its carve-out — a Ryzen 5000U reported `dedicated` with
|
|
332
|
-
* over a gigabyte of "VRAM". Vulkan calls the same device INTEGRATED_GPU, but
|
|
333
|
-
* that is not in the collector, so an AMD GPU on linux cannot be placed and
|
|
334
|
-
* assesses as `unknown`.
|
|
27
|
+
* Whether the source names one shard of a split model, read from its file name
|
|
28
|
+
* as `loadModel` reads it. A catalog constant names its file by registry path.
|
|
335
29
|
*/
|
|
336
|
-
function
|
|
337
|
-
if (
|
|
30
|
+
function isSplitSource(modelSrc) {
|
|
31
|
+
if (modelSrc === undefined)
|
|
338
32
|
return false;
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
kind: 'shared',
|
|
381
|
-
backend,
|
|
382
|
-
...(shared.name.status === 'supported' && { device: shared.name.value })
|
|
383
|
-
};
|
|
384
|
-
}
|
|
385
|
-
const samples = resources.sample?.gpus;
|
|
386
|
-
if (samples?.status !== 'supported')
|
|
387
|
-
return undefined;
|
|
388
|
-
const targets = [];
|
|
389
|
-
for (const gpu of deviceBacked) {
|
|
390
|
-
const backend = backendOf(gpu);
|
|
391
|
-
if (!backend)
|
|
392
|
-
return undefined;
|
|
393
|
-
const sample = samples.value.find((entry) => entry.id === gpu.id);
|
|
394
|
-
if (!sample)
|
|
395
|
-
return undefined;
|
|
396
|
-
if (sample.memoryTotalBytes.status !== 'supported')
|
|
397
|
-
return undefined;
|
|
398
|
-
if (sample.memoryUsedBytes.status !== 'supported')
|
|
399
|
-
return undefined;
|
|
400
|
-
if (sample.memoryTotalBytes.value <= 0)
|
|
401
|
-
return undefined;
|
|
402
|
-
if (sample.memoryUsedBytes.value > sample.memoryTotalBytes.value)
|
|
403
|
-
return undefined;
|
|
404
|
-
targets.push({
|
|
405
|
-
backend,
|
|
406
|
-
totalBytes: sample.memoryTotalBytes.value,
|
|
407
|
-
usedBytes: sample.memoryUsedBytes.value,
|
|
408
|
-
scope: sample.memoryTotalBytes.provenance.scope === 'budget' ? 'budget' : 'device',
|
|
409
|
-
...(gpu.name.status === 'supported' && { device: gpu.name.value })
|
|
33
|
+
const location = typeof modelSrc === 'string' ? modelSrc : (modelSrc.registryPath ?? modelSrc.src);
|
|
34
|
+
return detectShardedModel(path.basename(location)).isSharded;
|
|
35
|
+
}
|
|
36
|
+
/** Why a load with no description staged for it has no fitter verdict. */
|
|
37
|
+
export function noStubDetail(reason, message) {
|
|
38
|
+
if (reason === 'files-only-engine') {
|
|
39
|
+
return 'the engine measures downloaded files only, and this model is not downloaded';
|
|
40
|
+
}
|
|
41
|
+
return message === undefined
|
|
42
|
+
? `no registry description (${reason})`
|
|
43
|
+
: `no registry description (${reason}): ${message}`;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The fitter's answer for one candidate, run against the model on disk or the
|
|
47
|
+
* registry's weightless description of it.
|
|
48
|
+
*/
|
|
49
|
+
async function fitCandidate(candidate) {
|
|
50
|
+
const modelType = modelTypeOf(candidate);
|
|
51
|
+
if (!isCanonicalModelType(modelType))
|
|
52
|
+
throw new PluginNotFoundError(modelType);
|
|
53
|
+
const budgetMs = getConfig().fitStubBudgetMs;
|
|
54
|
+
const resolved = await resolveLoadFromStubs({
|
|
55
|
+
modelType,
|
|
56
|
+
...(candidate.modelSrc !== undefined && { modelSrc: candidate.modelSrc }),
|
|
57
|
+
...(candidate.modelConfig !== undefined && { modelConfig: candidate.modelConfig }),
|
|
58
|
+
...(candidate.transcriptionWorkload !== undefined && {
|
|
59
|
+
transcriptionWorkload: candidate.transcriptionWorkload
|
|
60
|
+
}),
|
|
61
|
+
...(candidate.ttsWorkload !== undefined && { ttsWorkload: candidate.ttsWorkload })
|
|
62
|
+
}, budgetMs !== undefined ? { budgetMs } : {});
|
|
63
|
+
if (resolved.status === 'no-stub') {
|
|
64
|
+
return unknownFit(noStubDetail(resolved.reason, resolved.message));
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
return await fitLoad(modelType, {
|
|
68
|
+
modelPath: resolved.modelPath,
|
|
69
|
+
modelConfig: resolved.modelConfig,
|
|
70
|
+
artifacts: resolved.artifacts,
|
|
71
|
+
isShardedModel: isSplitSource(candidate.modelSrc),
|
|
72
|
+
transcriptionWorkload: candidate.transcriptionWorkload,
|
|
73
|
+
ttsWorkload: candidate.ttsWorkload
|
|
410
74
|
});
|
|
411
75
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
const backend = targets[0].backend;
|
|
415
|
-
if (targets.some((target) => target.backend !== backend))
|
|
416
|
-
return undefined;
|
|
417
|
-
if (targets.some((target) => target.scope !== targets[0].scope))
|
|
418
|
-
return undefined;
|
|
419
|
-
return { kind: 'device', backend, targets };
|
|
420
|
-
}
|
|
421
|
-
function gpuBudget(target, platform) {
|
|
422
|
-
const available = target.totalBytes - target.usedBytes;
|
|
423
|
-
const reserved = reserveBytes(available, platform);
|
|
424
|
-
return {
|
|
425
|
-
totalBytes: target.totalBytes,
|
|
426
|
-
usedBytes: target.usedBytes,
|
|
427
|
-
availableBytes: available,
|
|
428
|
-
reservedBytes: reserved,
|
|
429
|
-
availableAfterReserveBytes: available - reserved
|
|
430
|
-
};
|
|
431
|
-
}
|
|
432
|
-
/** The budget with the least room left, which is the one worth reporting. */
|
|
433
|
-
function tightest(budgets) {
|
|
434
|
-
return budgets.reduce((least, budget) => budget.availableAfterReserveBytes < least.availableAfterReserveBytes ? budget : least);
|
|
435
|
-
}
|
|
436
|
-
/**
|
|
437
|
-
* What the candidate budgets have to hold. `alsoBoundBy` is present only where
|
|
438
|
-
* they are a card's own memory, which host bytes never come out of; without it
|
|
439
|
-
* they are system memory and everything lands there.
|
|
440
|
-
*/
|
|
441
|
-
function chargedToDevice(demand, alsoBoundBy) {
|
|
442
|
-
return alsoBoundBy ? demand.deviceBytes : totalBytes(demand);
|
|
443
|
-
}
|
|
444
|
-
/**
|
|
445
|
-
* `candidates` are alternatives — the engine pins the model to one of them, and
|
|
446
|
-
* which one is not observable here — so a fit has to hold on the smallest and a
|
|
447
|
-
* refusal on the largest. `alsoBoundBy` is a conjunction: a GPU load is paid
|
|
448
|
-
* for in system RAM too, so both bounds apply.
|
|
449
|
-
*/
|
|
450
|
-
function verdictAgainst(demand, candidates, alsoBoundBy) {
|
|
451
|
-
const room = candidates.map((budget) => budget.availableAfterReserveBytes);
|
|
452
|
-
const onDevice = chargedToDevice(demand, alsoBoundBy);
|
|
453
|
-
const primary = onDevice <= Math.min(...room)
|
|
454
|
-
? 'likely-fits'
|
|
455
|
-
: onDevice > Math.max(...room)
|
|
456
|
-
? 'likely-too-large'
|
|
457
|
-
: 'unknown';
|
|
458
|
-
if (!alsoBoundBy)
|
|
459
|
-
return primary;
|
|
460
|
-
return worst(primary, totalBytes(demand) > alsoBoundBy.availableAfterReserveBytes ? 'likely-too-large' : 'likely-fits');
|
|
461
|
-
}
|
|
462
|
-
/** The more pessimistic of two verdicts. */
|
|
463
|
-
function worst(a, b) {
|
|
464
|
-
if (a === 'likely-too-large' || b === 'likely-too-large')
|
|
465
|
-
return 'likely-too-large';
|
|
466
|
-
if (a === 'unknown' || b === 'unknown')
|
|
467
|
-
return 'unknown';
|
|
468
|
-
return 'likely-fits';
|
|
469
|
-
}
|
|
470
|
-
/**
|
|
471
|
-
* Picks the budget basis for a platform.
|
|
472
|
-
*
|
|
473
|
-
* iOS is the exception: jetsam terminates an app on its own footprint against
|
|
474
|
-
* a per-process limit well below device RAM, so a system-memory budget there
|
|
475
|
-
* would defend `likely-fits` verdicts the OS does not honor. Android's
|
|
476
|
-
* low-memory killer acts system-wide and native allocations carry no
|
|
477
|
-
* per-process cap, so it deliberately keeps the system basis with the mobile
|
|
478
|
-
* reserve.
|
|
479
|
-
*/
|
|
480
|
-
export function resolveBasis(platform) {
|
|
481
|
-
return platform === 'ios-arm64' ? 'process-memory' : 'system-memory';
|
|
482
|
-
}
|
|
483
|
-
function basisEvidence(basis) {
|
|
484
|
-
if (basis === 'process-memory')
|
|
485
|
-
return 'this process’s own memory ceiling';
|
|
486
|
-
if (basis === 'device-memory')
|
|
487
|
-
return 'the GPU’s own memory';
|
|
488
|
-
if (basis === 'device-budget')
|
|
489
|
-
return 'the GPU memory this process is budgeted';
|
|
490
|
-
return 'system memory';
|
|
491
|
-
}
|
|
492
|
-
/**
|
|
493
|
-
* Derives the memory budget from the sample, under the platform's basis.
|
|
494
|
-
*
|
|
495
|
-
* Only `sample.memory` is used: capabilities-only totals say nothing about what
|
|
496
|
-
* is free right now, and a verdict without that is not worth giving. Under the
|
|
497
|
-
* process basis the ceiling is reconstructed as allowance + footprint — the
|
|
498
|
-
* relation the OS enforces — so every budget field keeps the same meaning
|
|
499
|
-
* under either basis.
|
|
500
|
-
*/
|
|
501
|
-
export function resolveBudget(resources, platform, basis, reasons) {
|
|
502
|
-
const sample = resources.sample;
|
|
503
|
-
if (!sample) {
|
|
504
|
-
reasons.push('no memory sample was available');
|
|
505
|
-
return undefined;
|
|
506
|
-
}
|
|
507
|
-
if (basis === 'process-memory') {
|
|
508
|
-
const available = sample.memory.processAvailableBytes;
|
|
509
|
-
const used = sample.memory.processUsedBytes;
|
|
510
|
-
if (available.status !== 'supported' || used.status !== 'supported') {
|
|
511
|
-
reasons.push('iOS budgets are per-process (jetsam terminates on the app’s own footprint), and the per-process allowance metric is not available on this build');
|
|
512
|
-
return undefined;
|
|
513
|
-
}
|
|
514
|
-
const total = available.value + used.value;
|
|
515
|
-
if (total <= 0) {
|
|
516
|
-
reasons.push('process-memory metrics are inconsistent');
|
|
517
|
-
return undefined;
|
|
518
|
-
}
|
|
519
|
-
const reserved = reserveBytes(available.value, platform);
|
|
520
|
-
return {
|
|
521
|
-
totalBytes: total,
|
|
522
|
-
usedBytes: used.value,
|
|
523
|
-
availableBytes: available.value,
|
|
524
|
-
reservedBytes: reserved,
|
|
525
|
-
availableAfterReserveBytes: available.value - reserved
|
|
526
|
-
};
|
|
527
|
-
}
|
|
528
|
-
const total = sample.memory.totalBytes;
|
|
529
|
-
const used = sample.memory.usedBytes;
|
|
530
|
-
if (total.status !== 'supported' || used.status !== 'supported') {
|
|
531
|
-
reasons.push('system-memory metrics are not supported on this platform');
|
|
532
|
-
return undefined;
|
|
76
|
+
finally {
|
|
77
|
+
await resolved.release();
|
|
533
78
|
}
|
|
534
|
-
if (total.value <= 0 || used.value > total.value) {
|
|
535
|
-
reasons.push('system-memory metrics are inconsistent');
|
|
536
|
-
return undefined;
|
|
537
|
-
}
|
|
538
|
-
const available = total.value - used.value;
|
|
539
|
-
const reserved = reserveBytes(available, platform);
|
|
540
|
-
return {
|
|
541
|
-
totalBytes: total.value,
|
|
542
|
-
usedBytes: used.value,
|
|
543
|
-
availableBytes: available,
|
|
544
|
-
reservedBytes: reserved,
|
|
545
|
-
availableAfterReserveBytes: available - reserved
|
|
546
|
-
};
|
|
547
79
|
}
|
|
548
80
|
/**
|
|
549
|
-
*
|
|
550
|
-
* desktop and 1 GiB on mobile.
|
|
81
|
+
* Whether a set of models fits when loaded together.
|
|
551
82
|
*
|
|
552
|
-
*
|
|
553
|
-
*
|
|
554
|
-
* host it double-counts and can exceed the whole headroom — a 24 GiB Mac with
|
|
555
|
-
* 3.3 GiB free ended up with a zero budget and called a 2 GiB model too large.
|
|
556
|
-
*/
|
|
557
|
-
function reserveBytes(availableBytes, platform) {
|
|
558
|
-
const mobile = platform !== undefined && MOBILE_PLATFORMS.includes(platform);
|
|
559
|
-
return Math.min(mobile ? 1 * GIB : 2 * GIB, Math.floor(availableBytes * 0.2));
|
|
560
|
-
}
|
|
561
|
-
/**
|
|
562
|
-
* The combined lower bound when at least one candidate has only a floor: every
|
|
563
|
-
* estimate contributes its own lower bound, every floor its bytes, aggregated
|
|
564
|
-
* the same way as `aggregate` — a floor has no working peak to add.
|
|
83
|
+
* Each model's fitter runs in turn, so a set never holds several fitters open
|
|
84
|
+
* at once, and the answers are combined per memory pool against what is free.
|
|
565
85
|
*
|
|
566
|
-
*
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
* The combined demand when every candidate was measured by its own engine.
|
|
591
|
-
* Each fitter answered about one model against the whole machine, so only the
|
|
592
|
-
* bytes compose: persistent is always resident, and the working peak is summed
|
|
593
|
-
* under `concurrent` and maximised under `sequential`.
|
|
594
|
-
*/
|
|
595
|
-
function aggregateNative(natives, execution) {
|
|
596
|
-
let persistent = 0;
|
|
597
|
-
let working = 0;
|
|
598
|
-
let hostBytes = 0;
|
|
599
|
-
for (const native of natives) {
|
|
600
|
-
persistent += native.persistent;
|
|
601
|
-
hostBytes += native.hostBytes;
|
|
602
|
-
working =
|
|
603
|
-
execution === 'concurrent' ? working + native.working : Math.max(working, native.working);
|
|
604
|
-
}
|
|
605
|
-
return { deviceBytes: persistent + working, hostBytes };
|
|
606
|
-
}
|
|
607
|
-
/**
|
|
608
|
-
* A floor can only refuse: over the largest candidate budget it is
|
|
609
|
-
* `likely-too-large`, and anything else is `unknown`, because the cost above
|
|
610
|
-
* the floor is unmeasured and could be anything.
|
|
611
|
-
*/
|
|
612
|
-
function floorVerdict(floor, candidates, alsoBoundBy) {
|
|
613
|
-
const room = candidates.map((budget) => budget.availableAfterReserveBytes);
|
|
614
|
-
if (chargedToDevice(floor, alsoBoundBy) > Math.max(...room))
|
|
615
|
-
return 'likely-too-large';
|
|
616
|
-
if (alsoBoundBy && totalBytes(floor) > alsoBoundBy.availableAfterReserveBytes) {
|
|
617
|
-
return 'likely-too-large';
|
|
618
|
-
}
|
|
619
|
-
return 'unknown';
|
|
620
|
-
}
|
|
621
|
-
function toModelResult(candidate, result, budget,
|
|
622
|
-
/** Every candidate GPU budget, when the host has more than one. */
|
|
623
|
-
deviceBudgets,
|
|
624
|
-
/** The system budget a GPU load is also paid for out of. */
|
|
625
|
-
alsoBoundBy) {
|
|
626
|
-
if (result.kind === 'unknown') {
|
|
627
|
-
return {
|
|
628
|
-
name: candidate.model.name,
|
|
629
|
-
verdict: 'unknown',
|
|
630
|
-
...(candidate.device !== undefined && { device: candidate.device }),
|
|
631
|
-
reasons: [...result.reasons]
|
|
632
|
-
};
|
|
633
|
-
}
|
|
634
|
-
if (result.kind === 'floor') {
|
|
635
|
-
const verdict = budget
|
|
636
|
-
? floorVerdict({ deviceBytes: result.bytes, hostBytes: 0 }, deviceBudgets ?? [budget], alsoBoundBy)
|
|
637
|
-
: 'unknown';
|
|
638
|
-
return {
|
|
639
|
-
name: candidate.model.name,
|
|
640
|
-
verdict,
|
|
641
|
-
evidence: 'computed-only',
|
|
642
|
-
...(candidate.device !== undefined && { device: candidate.device }),
|
|
643
|
-
floorBytes: result.bytes,
|
|
644
|
-
estimatorVersion: FLOOR_VERSION,
|
|
645
|
-
reasons: [
|
|
646
|
-
...result.reasons,
|
|
647
|
-
!budget
|
|
648
|
-
? 'no usable system-memory sample, so this model has no verdict'
|
|
649
|
-
: verdict === 'likely-too-large'
|
|
650
|
-
? 'the computed floor alone exceeds the budget'
|
|
651
|
-
: 'the computed floor is within the budget, but the cost above it is unmeasured, so a fit cannot be claimed'
|
|
652
|
-
]
|
|
653
|
-
};
|
|
654
|
-
}
|
|
655
|
-
if (result.kind === 'bare') {
|
|
656
|
-
return {
|
|
657
|
-
name: candidate.model.name,
|
|
658
|
-
verdict: result.verdict === 'fit' ? 'likely-fits' : 'likely-too-large',
|
|
659
|
-
evidence: 'native-fit',
|
|
660
|
-
...(candidate.device !== undefined && { device: candidate.device }),
|
|
661
|
-
estimatorVersion: result.estimatorVersion,
|
|
662
|
-
reasons: [...result.reasons]
|
|
663
|
-
};
|
|
664
|
-
}
|
|
665
|
-
const bytes = aggregateNative([result], 'sequential');
|
|
666
|
-
const verdict = result.verdict === 'does-not-fit'
|
|
667
|
-
? 'likely-too-large'
|
|
668
|
-
: budget
|
|
669
|
-
? verdictAgainst(bytes, deviceBudgets ?? [budget], alsoBoundBy)
|
|
670
|
-
: 'likely-fits';
|
|
671
|
-
return {
|
|
672
|
-
name: candidate.model.name,
|
|
673
|
-
verdict,
|
|
674
|
-
evidence: 'native-fit',
|
|
675
|
-
...(candidate.device !== undefined && { device: candidate.device }),
|
|
676
|
-
estimatorVersion: result.estimatorVersion,
|
|
677
|
-
reasons: [...result.reasons]
|
|
678
|
-
};
|
|
86
|
+
* A load that `loadModel` would refuse throws the error that load would throw:
|
|
87
|
+
* no plugin for the model type, a config its schema or plugin refuses, or the
|
|
88
|
+
* fitter rejecting the load. A load that would start but cannot be sized is
|
|
89
|
+
* `unknown`.
|
|
90
|
+
*
|
|
91
|
+
* @throws {ModelTypeRequiredError} When neither the model type nor the source
|
|
92
|
+
* names an engine.
|
|
93
|
+
* @throws {PluginNotFoundError} When no plugin handles the model type.
|
|
94
|
+
* @throws {PluginLoadConfigValidationFailedError} When the plugin's schema
|
|
95
|
+
* refuses the config.
|
|
96
|
+
* @throws The plugin's or the fitter's own error when either refuses the load.
|
|
97
|
+
*/
|
|
98
|
+
export async function assessModelFit(candidates) {
|
|
99
|
+
const fits = [];
|
|
100
|
+
for (const candidate of candidates)
|
|
101
|
+
fits.push(await fitCandidate(candidate));
|
|
102
|
+
const models = fits.map((fit) => fit.result);
|
|
103
|
+
const { verdict, pools } = combineFits(models);
|
|
104
|
+
const rows = fits.map((fit, index) => {
|
|
105
|
+
const why = fit.reason === undefined ? '' : ` (${fit.reason})`;
|
|
106
|
+
return { label: `model ${index + 1} ${fit.result.status}${why}`, devices: fit.result.devices };
|
|
107
|
+
});
|
|
108
|
+
getEngineLogger().info(`[model-fit] ${verdict} — ${describeAccounting(pools, rows)}`);
|
|
109
|
+
return { verdict, pools, models };
|
|
679
110
|
}
|