@qvac/inference 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +89 -41
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +5 -2
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -2
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +39 -37
- package/dist/api/index.js +38 -36
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +6 -5
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +9 -0
- package/dist/errors/index.js +18 -1
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +84 -32
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -5
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +7 -5
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
- package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
- package/dist/model-fit/floor.d.ts +31 -0
- package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
- package/dist/model-fit/kv-cache.d.ts +23 -0
- package/dist/model-fit/kv-cache.js +85 -0
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
- package/dist/model-fit/platform.d.ts +14 -0
- package/dist/model-fit/platform.js +26 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
- package/dist/model-fit/request/create.js +61 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/model-fit/request/engines/audiogen.js +63 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
- package/dist/model-fit/request/engines/llama.js +44 -0
- package/dist/model-fit/request/engines/nmt.d.ts +8 -0
- package/dist/model-fit/request/engines/nmt.js +11 -0
- package/dist/model-fit/request/engines/ocr.d.ts +9 -0
- package/dist/model-fit/request/engines/ocr.js +13 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1072 -815
- package/dist/models/registry/models.js +3704 -3405
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +406 -3
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
- package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
- package/dist/plugins/builtin/index.d.ts +15 -12
- package/dist/plugins/builtin/index.js +15 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
- package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
- package/dist/plugins/ops/kv-cache-utils.js +25 -33
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +33 -19
- package/dist/plugins/ops/transcribe.js +18 -2
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +3 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +9 -1
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +22 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +7 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +106 -43
- package/dist/schemas/assess-model-fit.js +69 -32
- package/dist/schemas/audio-gen.d.ts +199 -19
- package/dist/schemas/audio-gen.js +183 -51
- package/dist/schemas/batch-completion-stream.d.ts +210 -36
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +3 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +24 -3
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +1465 -162
- package/dist/schemas/common.js +37 -31
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +243 -39
- package/dist/schemas/completion-stream.js +100 -29
- package/dist/schemas/config.d.ts +1326 -18
- package/dist/schemas/config.js +30 -8
- package/dist/schemas/decision.d.ts +610 -0
- package/dist/schemas/decision.js +219 -0
- package/dist/schemas/download-asset.d.ts +10 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +9 -1
- package/dist/schemas/engine-addon-map.js +26 -5
- package/dist/schemas/errors.d.ts +2 -0
- package/dist/schemas/errors.js +11 -1
- package/dist/schemas/index.d.ts +51 -48
- package/dist/schemas/index.js +50 -47
- package/dist/schemas/llamacpp-config.d.ts +222 -6
- package/dist/schemas/llamacpp-config.js +129 -7
- package/dist/schemas/load-model.d.ts +7960 -1492
- package/dist/schemas/load-model.js +122 -10
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +428 -9
- package/dist/schemas/model-config-schemas.js +16 -11
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +4 -3
- package/dist/schemas/model-resource-profile.d.ts +4 -4
- package/dist/schemas/model-resource-profile.js +2 -5
- package/dist/schemas/model-src-utils.d.ts +14 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +14 -1
- package/dist/schemas/model-types.js +4 -0
- package/dist/schemas/ocr.d.ts +76 -0
- package/dist/schemas/ocr.js +43 -1
- package/dist/schemas/plugin.d.ts +16 -4
- package/dist/schemas/plugin.js +12 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +34 -0
- package/dist/schemas/registry.js +5 -1
- package/dist/schemas/reload-config.d.ts +14 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +82 -0
- package/dist/schemas/sdcpp-config.js +41 -22
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +463 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +20 -0
- package/dist/schemas/transcription-config.js +54 -4
- package/dist/schemas/transcription.d.ts +47 -5
- package/dist/schemas/transcription.js +52 -5
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +40 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +3 -2
- package/dist/surface.js +3 -2
- package/dist/utils/addon-responses.d.ts +3 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +13 -9
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +2 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +68 -51
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -28
- package/dist/resources/model-fit/types.d.ts +0 -120
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { NativeProbeFit } from '
|
|
2
|
-
import type { FitProbeResult } from '
|
|
1
|
+
import type { NativeProbeFit } from '../schemas/index.js';
|
|
2
|
+
import type { FitProbeResult } from '../model-fit/run/engine.js';
|
|
3
3
|
export declare function classifyFit(probe: FitProbeResult, provenance: {
|
|
4
4
|
basis: 'native-probe';
|
|
5
5
|
estimatorVersion: string;
|
|
@@ -17,15 +17,15 @@ function verdictFor(status) {
|
|
|
17
17
|
return 'unknown';
|
|
18
18
|
}
|
|
19
19
|
/**
|
|
20
|
-
* The diffusion fitter reports a placement
|
|
21
|
-
*
|
|
22
|
-
*
|
|
20
|
+
* The diffusion fitter reports a placement alongside its reason. `changed`
|
|
21
|
+
* means it reached a fitting arrangement only by moving modules between
|
|
22
|
+
* backends, which is why the status is already `does-not-fit`.
|
|
23
23
|
*/
|
|
24
24
|
function diffusionReason(result) {
|
|
25
25
|
if (result.status === 'fits')
|
|
26
26
|
return 'fits';
|
|
27
27
|
if (result.status === 'error')
|
|
28
|
-
return
|
|
28
|
+
return result.reason;
|
|
29
29
|
return result.changed ? `does-not-fit-as-configured (${result.backend})` : 'does-not-fit';
|
|
30
30
|
}
|
|
31
31
|
function reasonFor(probe) {
|
|
@@ -45,7 +45,8 @@ function breakdown(weights, context, compute) {
|
|
|
45
45
|
/**
|
|
46
46
|
* The parts of `deviceBytes` the engine separates. Audiogen reports a peak
|
|
47
47
|
* across pipeline phases and diffusion a per-module table, neither of which
|
|
48
|
-
* divides into these three, so both come back empty.
|
|
48
|
+
* divides into these three, so both come back empty. OCR sizes its weights
|
|
49
|
+
* alone and leaves the image-dependent memory uncounted.
|
|
49
50
|
*
|
|
50
51
|
* The llama rows are summed over the same devices the engine's own
|
|
51
52
|
* `deviceBytes` covers, which is every row but `host`.
|
|
@@ -71,14 +72,28 @@ function breakdownFor(probe) {
|
|
|
71
72
|
const result = probe.result;
|
|
72
73
|
return breakdown(result.weightsBytes, result.stateBytes, sumOf(result.lmComputeBytes, result.codecComputeBytes));
|
|
73
74
|
}
|
|
75
|
+
if (probe.engine === 'ocr-ggml') {
|
|
76
|
+
return breakdown(probe.result.weightsBytes, undefined, undefined);
|
|
77
|
+
}
|
|
74
78
|
return {};
|
|
75
79
|
}
|
|
76
80
|
function llamaProjection(probe) {
|
|
77
81
|
const first = probe.result.devices.find((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE);
|
|
82
|
+
const devices = probe.result.devices
|
|
83
|
+
.filter((device) => device.name !== HOST_ROW)
|
|
84
|
+
.map((device) => ({
|
|
85
|
+
name: device.name,
|
|
86
|
+
totalBytes: device.totalBytes,
|
|
87
|
+
freeBytes: device.freeBytes,
|
|
88
|
+
weightsBytes: device.modelBytes,
|
|
89
|
+
contextBytes: device.contextBytes,
|
|
90
|
+
computeBytes: device.computeBytes
|
|
91
|
+
}));
|
|
78
92
|
return {
|
|
79
93
|
deviceBytes: probe.result.deviceBytes,
|
|
80
94
|
hostBytes: probe.result.hostBytes,
|
|
81
95
|
...breakdownFor(probe),
|
|
96
|
+
...(devices.length > 0 && { devices }),
|
|
82
97
|
...(first !== undefined && {
|
|
83
98
|
deviceName: first.name,
|
|
84
99
|
deviceFreeBytes: first.freeBytes,
|
|
@@ -86,10 +101,26 @@ function llamaProjection(probe) {
|
|
|
86
101
|
})
|
|
87
102
|
};
|
|
88
103
|
}
|
|
104
|
+
/**
|
|
105
|
+
* The translation fitter reports one allowance against the free memory of the
|
|
106
|
+
* device it ran on. Only the file bytes inside it are weights; the rest does
|
|
107
|
+
* not divide into context and compute.
|
|
108
|
+
*/
|
|
109
|
+
function nmtProjection(result) {
|
|
110
|
+
return {
|
|
111
|
+
deviceName: result.backend,
|
|
112
|
+
deviceBytes: result.requiredBytes,
|
|
113
|
+
deviceFreeBytes: result.freeBytes,
|
|
114
|
+
weightsBytes: result.modelBytes,
|
|
115
|
+
report: result.report
|
|
116
|
+
};
|
|
117
|
+
}
|
|
89
118
|
function projectionFor(probe) {
|
|
90
119
|
if (probe.engine === 'diffusion-cpp') {
|
|
91
120
|
return { deviceName: probe.result.backend, report: probe.result.report };
|
|
92
121
|
}
|
|
122
|
+
if (probe.engine === 'translation-nmtcpp')
|
|
123
|
+
return nmtProjection(probe.result);
|
|
93
124
|
if (isLlama(probe))
|
|
94
125
|
return llamaProjection(probe);
|
|
95
126
|
const result = probe.result;
|
package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { QVACBlobBinding } from '@qvac/registry-client';
|
|
3
|
-
import type { Logger } from '
|
|
3
|
+
import type { Logger } from '../../logging/types.js';
|
|
4
4
|
/**
|
|
5
5
|
* A weightless description of an artifact, served by the registry beside the
|
|
6
6
|
* weights: for a GGUF, the tensor list and the settings with the tokenizer
|
|
@@ -30,7 +30,9 @@ export type FitStubUnavailableReason =
|
|
|
30
30
|
/** The entry predates fit blobs, or its description could not be built. */
|
|
31
31
|
| 'no-fit-blob' | 'download-failed'
|
|
32
32
|
/** The lookup and the fetch together did not finish within the budget. */
|
|
33
|
-
| 'timed-out'
|
|
33
|
+
| 'timed-out'
|
|
34
|
+
/** The engine's fitter reads the files themselves and no registry description. */
|
|
35
|
+
| 'files-only-engine';
|
|
34
36
|
export type FitStubOutcome =
|
|
35
37
|
/** `path` is the caller's to remove once the fitter has read it. */
|
|
36
38
|
{
|
package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js}
RENAMED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import fs, { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
3
|
import { AbortController } from 'bare-abort-controller';
|
|
4
|
-
import { getCacheDir } from '
|
|
5
|
-
import { calculateFileChecksum } from '
|
|
6
|
-
import { getEngineLogger } from '
|
|
4
|
+
import { getCacheDir } from '../../utils/cache/paths.js';
|
|
5
|
+
import { calculateFileChecksum } from '../../utils/checksum.js';
|
|
6
|
+
import { getEngineLogger } from '../../logging/index.js';
|
|
7
7
|
/**
|
|
8
8
|
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
9
9
|
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
@@ -20,12 +20,12 @@ import { getEngineLogger } from '../../../logging/index.js';
|
|
|
20
20
|
*/
|
|
21
21
|
export const FIT_STUB_BUDGET_MS = 40_000;
|
|
22
22
|
async function defaultGetEntry(registryPath, registrySource) {
|
|
23
|
-
const { getRegistryClient } = await import('
|
|
23
|
+
const { getRegistryClient } = await import('../../runtime/registry-client.js');
|
|
24
24
|
const client = await getRegistryClient();
|
|
25
25
|
return client.getModel(registryPath, registrySource);
|
|
26
26
|
}
|
|
27
27
|
async function defaultDownloadBlob(binding, outputFile, signal) {
|
|
28
|
-
const { getRegistryClient } = await import('
|
|
28
|
+
const { getRegistryClient } = await import('../../runtime/registry-client.js');
|
|
29
29
|
const client = await getRegistryClient();
|
|
30
30
|
return client.downloadBlob(binding, {
|
|
31
31
|
outputFile,
|
|
@@ -94,7 +94,12 @@ function withinBudget(name, options, work) {
|
|
|
94
94
|
async function makeStagingDir(options) {
|
|
95
95
|
const root = options.cacheDir ?? getCacheDir('fit-stubs');
|
|
96
96
|
await fsPromises.mkdir(root, { recursive: true });
|
|
97
|
-
|
|
97
|
+
// On Windows `mkdtemp` returns the `\\?\` namespaced form of the path, and
|
|
98
|
+
// an engine that derives a companion's path by appending `/<name>` to it
|
|
99
|
+
// (bci's colocated embedder) then names a file that does not resolve.
|
|
100
|
+
// Keep the root as given and take only the generated directory name.
|
|
101
|
+
const created = await fsPromises.mkdtemp(path.join(root, 'stub-'));
|
|
102
|
+
return path.join(root, path.basename(created));
|
|
98
103
|
}
|
|
99
104
|
async function bindingFor(ref, options) {
|
|
100
105
|
const { registryPath, registrySource } = ref;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { type LoadDescription } from '
|
|
2
|
-
import type { FitStubOptions, FitStubUnavailableReason } from '
|
|
3
|
-
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '
|
|
1
|
+
import { type LoadDescription } from '../../model-fit/description/resolve-load.js';
|
|
2
|
+
import type { FitStubOptions, FitStubUnavailableReason } from '../../model-fit/description/fetch.js';
|
|
3
|
+
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../../model-fit/load-check.js';
|
|
4
4
|
export type LoadFitOutcome = {
|
|
5
5
|
status: 'projected';
|
|
6
6
|
fit: AdvisoryFitOutcome;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { resolveLoadFromStubs } from '
|
|
2
|
-
import { runAdvisoryFitCheck } from '
|
|
1
|
+
import { resolveLoadFromStubs } from '../../model-fit/description/resolve-load.js';
|
|
2
|
+
import { runAdvisoryFitCheck } from '../../model-fit/load-check.js';
|
|
3
3
|
/**
|
|
4
4
|
* Projects the load a caller describes, before its weights exist locally.
|
|
5
5
|
*
|
|
@@ -18,6 +18,8 @@ export async function projectFitFromLoad(load, modelId, options = {}) {
|
|
|
18
18
|
modelType: load.modelType,
|
|
19
19
|
modelPath: resolved.modelPath,
|
|
20
20
|
modelConfig: resolved.modelConfig,
|
|
21
|
+
transcriptionWorkload: load.transcriptionWorkload,
|
|
22
|
+
ttsWorkload: load.ttsWorkload,
|
|
21
23
|
artifacts: resolved.artifacts,
|
|
22
24
|
// Nothing here assembles a shard set, so no load may claim the fitter
|
|
23
25
|
// is looking at a split model.
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type
|
|
3
|
-
import { type FitStubOptions, type FitStubUnavailableReason } from '
|
|
1
|
+
import type { TtsFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
import { type CanonicalModelType, type ModelSrcInput, type TranscriptionFitWorkload } from '../../schemas/index.js';
|
|
3
|
+
import { type FitStubOptions, type FitStubUnavailableReason } from '../../model-fit/description/fetch.js';
|
|
4
4
|
export interface LoadDescription {
|
|
5
|
+
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
6
|
+
ttsWorkload?: TtsFitWorkload | undefined;
|
|
5
7
|
/** Absent for the loads `loadModel` also takes without one. */
|
|
6
8
|
modelSrc?: ModelSrcInput | undefined;
|
|
7
9
|
modelType: CanonicalModelType;
|
|
@@ -33,6 +35,6 @@ export type LoadResolution = ResolvedLoad | {
|
|
|
33
35
|
* keys are the real load's.
|
|
34
36
|
*
|
|
35
37
|
* A source already on disk is read directly, since it describes the load more
|
|
36
|
-
* exactly than a stub does.
|
|
38
|
+
* exactly than a stub does. A files-only engine reads only sources on disk.
|
|
37
39
|
*/
|
|
38
40
|
export declare function resolveLoadFromStubs(load: LoadDescription, options?: FitStubOptions): Promise<LoadResolution>;
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import fs from 'bare-fs';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
2
|
+
import { ModelType } from '../../schemas/index.js';
|
|
3
|
+
import { getPlugin } from '../../plugins/registry.js';
|
|
4
|
+
import { getRuntimeContext } from '../../runtime/state.js';
|
|
5
|
+
import { formatZodError } from '../../utils/zod-error.js';
|
|
6
|
+
import { getModelByPath } from '../../models/registry/index.js';
|
|
7
|
+
import { fetchFitStub, fetchFitStubSet, removeStub } from '../../model-fit/description/fetch.js';
|
|
7
8
|
function srcString(modelSrc) {
|
|
8
9
|
return typeof modelSrc === 'string' ? modelSrc : modelSrc.src;
|
|
9
10
|
}
|
|
@@ -46,6 +47,11 @@ function stubSetFor(registryPath) {
|
|
|
46
47
|
}
|
|
47
48
|
return primary === undefined ? undefined : { primary, others };
|
|
48
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* Engines whose fitter sizes the files on disk and reads no registry
|
|
52
|
+
* description, so a stub would be answered for as if it were the model.
|
|
53
|
+
*/
|
|
54
|
+
const FILES_ONLY_ENGINES = new Set([ModelType.nmtcppTranslation]);
|
|
49
55
|
class NoStubError extends Error {
|
|
50
56
|
reason;
|
|
51
57
|
constructor(reason, message) {
|
|
@@ -63,7 +69,7 @@ class NoStubError extends Error {
|
|
|
63
69
|
* keys are the real load's.
|
|
64
70
|
*
|
|
65
71
|
* A source already on disk is read directly, since it describes the load more
|
|
66
|
-
* exactly than a stub does.
|
|
72
|
+
* exactly than a stub does. A files-only engine reads only sources on disk.
|
|
67
73
|
*/
|
|
68
74
|
export async function resolveLoadFromStubs(load, options = {}) {
|
|
69
75
|
const plugin = getPlugin(load.modelType);
|
|
@@ -84,6 +90,8 @@ export async function resolveLoadFromStubs(load, options = {}) {
|
|
|
84
90
|
const location = srcString(src);
|
|
85
91
|
if (location !== '' && fs.existsSync(location))
|
|
86
92
|
return location;
|
|
93
|
+
if (FILES_ONLY_ENGINES.has(load.modelType))
|
|
94
|
+
throw new NoStubError('files-only-engine');
|
|
87
95
|
const ref = stubRefFor(src);
|
|
88
96
|
if (ref === undefined)
|
|
89
97
|
throw new NoStubError('unresolvable-ref', location);
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ModelFitWorkload } from '../schemas/assess-model-fit.js';
|
|
2
|
+
import type { ModelResourceProfile } from '../schemas/model-resource-profile.js';
|
|
3
|
+
export declare const FLOOR_VERSION = "floor-v1";
|
|
4
|
+
export interface FloorInput {
|
|
5
|
+
profile: ModelResourceProfile;
|
|
6
|
+
workload: ModelFitWorkload;
|
|
7
|
+
/** Artifact bytes of companion constants the caller passed alongside. */
|
|
8
|
+
extraArtifactBytes: number;
|
|
9
|
+
}
|
|
10
|
+
/** A lower bound with no upper bound, and how it was assembled. */
|
|
11
|
+
export interface FloorResult {
|
|
12
|
+
bytes: number;
|
|
13
|
+
reasons: readonly string[];
|
|
14
|
+
assumptions: readonly string[];
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The smallest resident footprint the catalog facts prove: artifact bytes, plus
|
|
18
|
+
* the KV cache for a llama.cpp model at the cheapest element width the engine
|
|
19
|
+
* can default to.
|
|
20
|
+
*
|
|
21
|
+
* Every term the real load adds on top — weight slack, engine overhead, compute
|
|
22
|
+
* buffers, a completion's working peak — is non-negative, so a floor with all of
|
|
23
|
+
* them at zero is unconditionally valid on any platform and any backend. It can
|
|
24
|
+
* therefore refuse a model (`likely-too-large` when even the floor is over
|
|
25
|
+
* budget) but never confirm one, since how far above the floor the real load
|
|
26
|
+
* lands is unmeasured here.
|
|
27
|
+
*
|
|
28
|
+
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
29
|
+
* fallback wherever no engine measurement is available.
|
|
30
|
+
*/
|
|
31
|
+
export declare function computeFloor(input: FloorInput): FloorResult;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { LLAMA_WEIGHTS_ASSUMPTION,
|
|
1
|
+
import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheFloorBytes, narrowestKvElementBytes } from '../model-fit/kv-cache.js';
|
|
2
2
|
export const FLOOR_VERSION = 'floor-v1';
|
|
3
3
|
/**
|
|
4
4
|
* Engines whose load builds a KV cache the file sizes exactly, so the floor can
|
|
@@ -7,19 +7,19 @@ export const FLOOR_VERSION = 'floor-v1';
|
|
|
7
7
|
*/
|
|
8
8
|
const KV_CACHE_ENGINES = new Set(['llamacpp-completion', 'llamacpp-embedding']);
|
|
9
9
|
/**
|
|
10
|
-
* The smallest resident footprint the catalog facts prove
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* The smallest resident footprint the catalog facts prove: artifact bytes, plus
|
|
11
|
+
* the KV cache for a llama.cpp model at the cheapest element width the engine
|
|
12
|
+
* can default to.
|
|
13
13
|
*
|
|
14
|
-
* Every term
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
14
|
+
* Every term the real load adds on top — weight slack, engine overhead, compute
|
|
15
|
+
* buffers, a completion's working peak — is non-negative, so a floor with all of
|
|
16
|
+
* them at zero is unconditionally valid on any platform and any backend. It can
|
|
17
|
+
* therefore refuse a model (`likely-too-large` when even the floor is over
|
|
18
|
+
* budget) but never confirm one, since how far above the floor the real load
|
|
19
|
+
* lands is unmeasured here.
|
|
20
20
|
*
|
|
21
21
|
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
22
|
-
* fallback wherever no
|
|
22
|
+
* fallback wherever no engine measurement is available.
|
|
23
23
|
*/
|
|
24
24
|
export function computeFloor(input) {
|
|
25
25
|
const { profile, workload, extraArtifactBytes } = input;
|
|
@@ -55,8 +55,8 @@ export function computeFloor(input) {
|
|
|
55
55
|
// The narrowest default the engine can pick on any backend, so the floor
|
|
56
56
|
// holds whether the model lands on a GPU (q8_0 with flash attention) or on
|
|
57
57
|
// the CPU (f16). Architectures that disable flash attention stay f16 anyway.
|
|
58
|
-
const element =
|
|
59
|
-
const kv =
|
|
58
|
+
const element = narrowestKvElementBytes(facts);
|
|
59
|
+
const kv = kvCacheFloorBytes(facts, contextTokens, element);
|
|
60
60
|
assumptions.push(element < 2
|
|
61
61
|
? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
|
|
62
62
|
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { GgufFacts } from '../schemas/model-resource-profile.js';
|
|
2
|
+
export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
|
|
3
|
+
/**
|
|
4
|
+
* The narrowest KV-cache element width the engine can default to for this
|
|
5
|
+
* model, in bytes: q8_0 where flash attention allows it, f16 otherwise.
|
|
6
|
+
*/
|
|
7
|
+
export declare function narrowestKvElementBytes(facts: GgufFacts): number;
|
|
8
|
+
/**
|
|
9
|
+
* The smallest KV cache the requested context can need, at a fixed element
|
|
10
|
+
* width and clamped to the trained context.
|
|
11
|
+
*
|
|
12
|
+
* Three cases, in order of how much the file actually tells us:
|
|
13
|
+
*
|
|
14
|
+
* 1. **Per-layer classes** — the file describes attention per block, so the
|
|
15
|
+
* cache is summed exactly, with sliding-window blocks capped at their window.
|
|
16
|
+
* 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
|
|
17
|
+
* blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
|
|
18
|
+
* blocks are which is engine-owned, so the fewest full blocks are counted.
|
|
19
|
+
* 3. **Flat** — every block holds the same cache. A declared sliding window
|
|
20
|
+
* with no per-layer pattern is engine-owned, so every block is counted
|
|
21
|
+
* windowed.
|
|
22
|
+
*/
|
|
23
|
+
export declare function kvCacheFloorBytes(facts: GgufFacts, contextTokens: number, elementBytes: number): number;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KV-cache element widths, in bytes per element.
|
|
3
|
+
*
|
|
4
|
+
* `f16` is the CPU default. On a Metal/Vulkan GPU backend with flash attention
|
|
5
|
+
* on — the SDK's own defaults — `llm-llamacpp` defaults the cache to `q8_0`
|
|
6
|
+
* instead (`addon/src/model-interface/LoadFitNormalization.cpp`, QVAC-21318).
|
|
7
|
+
* `q8_0` packs 32 elements into a 34-byte block.
|
|
8
|
+
*/
|
|
9
|
+
const F16_BYTES_PER_ELEMENT = 2;
|
|
10
|
+
const Q8_0_BYTES_PER_ELEMENT = 34 / 32;
|
|
11
|
+
/** Recurrent/SSM state is kept in f32. */
|
|
12
|
+
const SSM_STATE_BYTES_PER_ELEMENT = 4;
|
|
13
|
+
/**
|
|
14
|
+
* Architectures that disable flash attention, so the GPU `q8_0` KV default
|
|
15
|
+
* never applies and the cache stays `f16` on every backend.
|
|
16
|
+
*/
|
|
17
|
+
function disablesFlashAttention(architecture) {
|
|
18
|
+
return architecture.startsWith('bitnet');
|
|
19
|
+
}
|
|
20
|
+
// Shared with the computed floor so a mixed set carries one weights assumption, not two.
|
|
21
|
+
export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
|
|
22
|
+
/**
|
|
23
|
+
* The narrowest KV-cache element width the engine can default to for this
|
|
24
|
+
* model, in bytes: q8_0 where flash attention allows it, f16 otherwise.
|
|
25
|
+
*/
|
|
26
|
+
export function narrowestKvElementBytes(facts) {
|
|
27
|
+
return disablesFlashAttention(facts.architecture) ? F16_BYTES_PER_ELEMENT : Q8_0_BYTES_PER_ELEMENT;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* The smallest KV cache the requested context can need, at a fixed element
|
|
31
|
+
* width and clamped to the trained context.
|
|
32
|
+
*
|
|
33
|
+
* Three cases, in order of how much the file actually tells us:
|
|
34
|
+
*
|
|
35
|
+
* 1. **Per-layer classes** — the file describes attention per block, so the
|
|
36
|
+
* cache is summed exactly, with sliding-window blocks capped at their window.
|
|
37
|
+
* 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
|
|
38
|
+
* blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
|
|
39
|
+
* blocks are which is engine-owned, so the fewest full blocks are counted.
|
|
40
|
+
* 3. **Flat** — every block holds the same cache. A declared sliding window
|
|
41
|
+
* with no per-layer pattern is engine-owned, so every block is counted
|
|
42
|
+
* windowed.
|
|
43
|
+
*/
|
|
44
|
+
export function kvCacheFloorBytes(facts, contextTokens, elementBytes) {
|
|
45
|
+
const tokens = Math.min(contextTokens, facts.contextLength);
|
|
46
|
+
if (facts.kvLayerClasses && facts.kvLayerClasses.length > 0) {
|
|
47
|
+
return layerClassBytes(facts.kvLayerClasses, facts, tokens, elementBytes);
|
|
48
|
+
}
|
|
49
|
+
const perBlockPerToken = facts.headCountKv * (facts.keyLength + facts.valueLength);
|
|
50
|
+
if (facts.fullAttentionInterval && facts.fullAttentionInterval > 1) {
|
|
51
|
+
const fullBlocks = Math.floor(facts.blockCount / facts.fullAttentionInterval);
|
|
52
|
+
const ssm = ssmStateBytes(facts, facts.blockCount - fullBlocks);
|
|
53
|
+
return fullBlocks * perBlockPerToken * tokens * elementBytes + ssm;
|
|
54
|
+
}
|
|
55
|
+
if (facts.slidingWindow) {
|
|
56
|
+
const windowedTokens = Math.min(tokens, facts.slidingWindow);
|
|
57
|
+
return facts.blockCount * perBlockPerToken * windowedTokens * elementBytes;
|
|
58
|
+
}
|
|
59
|
+
return facts.blockCount * perBlockPerToken * tokens * elementBytes;
|
|
60
|
+
}
|
|
61
|
+
function layerClassBytes(classes, facts, contextTokens, elementBytes) {
|
|
62
|
+
let total = 0;
|
|
63
|
+
for (const layerClass of classes) {
|
|
64
|
+
const tokens = layerClass.windowed && facts.slidingWindow
|
|
65
|
+
? Math.min(contextTokens, facts.slidingWindow)
|
|
66
|
+
: contextTokens;
|
|
67
|
+
total +=
|
|
68
|
+
layerClass.count *
|
|
69
|
+
layerClass.headCountKv *
|
|
70
|
+
(layerClass.keyLength + layerClass.valueLength) *
|
|
71
|
+
tokens *
|
|
72
|
+
elementBytes;
|
|
73
|
+
}
|
|
74
|
+
return total;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Fixed recurrent state for the blocks of a hybrid model that hold no KV cache:
|
|
78
|
+
* the SSM state plus its convolution window, per block.
|
|
79
|
+
*/
|
|
80
|
+
function ssmStateBytes(facts, recurrentBlocks) {
|
|
81
|
+
if (!facts.ssmInnerSize || !facts.ssmStateSize || recurrentBlocks <= 0)
|
|
82
|
+
return 0;
|
|
83
|
+
const perBlock = facts.ssmInnerSize * facts.ssmStateSize + facts.ssmInnerSize * (facts.ssmConvKernel ?? 0);
|
|
84
|
+
return recurrentBlocks * perBlock * SSM_STATE_BYTES_PER_ELEMENT;
|
|
85
|
+
}
|
package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts}
RENAMED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import type { Logger } from '
|
|
3
|
-
import type {
|
|
4
|
-
import {
|
|
2
|
+
import type { Logger } from '../logging/types.js';
|
|
3
|
+
import type { TtsFitWorkload } from '../schemas/assess-model-fit.js';
|
|
4
|
+
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict, TranscriptionFitWorkload } from '../schemas/index.js';
|
|
5
|
+
import { runFit as runFitDefault } from '../model-fit/run/run.js';
|
|
5
6
|
/**
|
|
6
|
-
* `fit` and `does-not-fit` are projections of the load the SDK is about to run
|
|
7
|
-
*
|
|
8
|
-
* fitted plan, so neither verdict is denial-grade and no verdict changes the
|
|
9
|
-
* load.
|
|
7
|
+
* `fit` and `does-not-fit` are projections of the load the SDK is about to run.
|
|
8
|
+
* The real loader neither consumes nor verifies the fitted plan.
|
|
10
9
|
*
|
|
11
10
|
* The outcome is the wire shape: `loadModel` hands it to the model registry and
|
|
12
11
|
* `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
|
|
@@ -15,6 +14,8 @@ import { runFit as runFitDefault } from './run-fit.js';
|
|
|
15
14
|
export type AdvisoryFitVerdict = NativeProbeVerdict;
|
|
16
15
|
export type AdvisoryFitOutcome = NativeProbeFit;
|
|
17
16
|
export interface AdvisoryFitInput {
|
|
17
|
+
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
18
|
+
ttsWorkload?: TtsFitWorkload | undefined;
|
|
18
19
|
modelId: string;
|
|
19
20
|
modelType: CanonicalModelType;
|
|
20
21
|
modelPath: string;
|
|
@@ -28,22 +29,24 @@ export interface AdvisoryFitInput {
|
|
|
28
29
|
*/
|
|
29
30
|
export interface AdvisoryFitOptions {
|
|
30
31
|
signal?: AbortSignal;
|
|
31
|
-
enabled?: boolean;
|
|
32
|
-
mobile?: boolean;
|
|
33
|
-
timeoutMs?: number;
|
|
34
32
|
runFit?: typeof runFitDefault;
|
|
35
33
|
logger?: Logger;
|
|
36
34
|
availableSystemBytes?: () => Promise<number | undefined>;
|
|
37
35
|
countsDeviceRows?: () => Promise<boolean>;
|
|
38
36
|
residentModelBytes?: () => Promise<number>;
|
|
37
|
+
/**
|
|
38
|
+
* Bytes to hold back beyond what is resident, for models the caller counts
|
|
39
|
+
* as loaded alongside this one but that this worker does not yet hold.
|
|
40
|
+
*/
|
|
41
|
+
extraResidentBytes?: number;
|
|
39
42
|
}
|
|
40
43
|
/**
|
|
41
44
|
* Runs the advisory fit check for a load that is about to start, against the
|
|
42
45
|
* fitter belonging to the engine that will run it.
|
|
43
46
|
*
|
|
44
|
-
* Fail-open by construction: an unsupported shape, a
|
|
45
|
-
*
|
|
46
|
-
* and the caller proceeds with the ordinary load path.
|
|
47
|
-
* throws and never rejects.
|
|
47
|
+
* Fail-open by construction: an unsupported shape, a fitter that crashed,
|
|
48
|
+
* threw or did not return in time, and an unexpected internal error all
|
|
49
|
+
* resolve to `unknown` and the caller proceeds with the ordinary load path.
|
|
50
|
+
* This function never throws and never rejects.
|
|
48
51
|
*/
|
|
49
52
|
export declare function runAdvisoryFitCheck(input: AdvisoryFitInput, options?: AdvisoryFitOptions): Promise<AdvisoryFitOutcome>;
|