@qvac/inference 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +59 -40
- package/README.md +1 -1
- package/dist/api/audio-gen.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +3 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/index.d.ts +39 -38
- package/dist/api/index.js +38 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +6 -0
- package/dist/errors/index.js +13 -1
- package/dist/handlers/assess-model-fit.d.ts +1 -1
- package/dist/handlers/assess-model-fit.js +4 -4
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +2 -3
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.js +4 -2
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +1 -1
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +4 -4
- package/dist/{resources/model-fit → model-fit}/assess.js +3 -3
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +4 -4
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +1 -1
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +5 -5
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +2 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +6 -6
- package/dist/{resources/model-fit → model-fit}/floor.d.ts +2 -2
- package/dist/{resources/model-fit → model-fit}/floor.js +1 -1
- package/dist/{resources/model-fit → model-fit}/kv-cache.d.ts +1 -1
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +8 -10
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +14 -40
- package/dist/{resources/model-fit/types.d.ts → model-fit/platform.d.ts} +8 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +3 -3
- package/dist/{resources/model-fit/native-probe/create-fit-request.js → model-fit/request/create.js} +8 -8
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +2 -2
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +9 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.js +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -2
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.js +13 -10
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/nmt.d.ts +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/nmt.js +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/ocr.d.ts +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/ocr.js +1 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +2 -2
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +1 -1
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +3 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +1 -1
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +510 -0
- package/dist/models/registry/models.js +2946 -2410
- package/dist/models/registry/resource-profiles.js +203 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +60 -54
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +4 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +26 -25
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +4 -4
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +11 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +4 -4
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +12 -12
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +15 -14
- package/dist/plugins/builtin/index.js +15 -14
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +36 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +169 -78
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -17
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +39 -33
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +7 -6
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +16 -6
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +6 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +16 -16
- package/dist/plugins/builtin/moss-transcribe/plugin.js +4 -4
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +18 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +29 -29
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +4 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +19 -8
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +90 -34
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +45 -45
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +2 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +2 -2
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +47 -32
- package/dist/plugins/builtin/tts-ggml/plugin.js +3 -3
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +34 -33
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +4 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +4 -3
- package/dist/plugins/ops/kv-cache-utils.js +15 -8
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +14 -5
- package/dist/plugins/ops/transcribe.js +1 -1
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +2 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +3 -3
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +3 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +14 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +6 -0
- package/dist/schemas/assess-model-fit.js +4 -4
- package/dist/schemas/audio-gen.d.ts +13 -1
- package/dist/schemas/audio-gen.js +3 -3
- package/dist/schemas/batch-completion-stream.d.ts +8 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +1 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +478 -100
- package/dist/schemas/common.js +35 -32
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +11 -2
- package/dist/schemas/completion-stream.js +37 -10
- package/dist/schemas/config.d.ts +849 -6
- package/dist/schemas/config.js +12 -10
- package/dist/schemas/decision.d.ts +8 -0
- package/dist/schemas/decision.js +14 -0
- package/dist/schemas/download-asset.d.ts +6 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +2 -1
- package/dist/schemas/engine-addon-map.js +4 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +6 -1
- package/dist/schemas/index.d.ts +51 -50
- package/dist/schemas/index.js +50 -49
- package/dist/schemas/llamacpp-config.d.ts +182 -6
- package/dist/schemas/llamacpp-config.js +102 -7
- package/dist/schemas/load-model.d.ts +3176 -646
- package/dist/schemas/load-model.js +34 -1
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +195 -1
- package/dist/schemas/model-config-schemas.js +14 -12
- package/dist/schemas/model-fit-policy.d.ts +1 -1
- package/dist/schemas/model-info.js +2 -2
- package/dist/schemas/model-resource-profile.d.ts +1 -0
- package/dist/schemas/model-resource-profile.js +1 -1
- package/dist/schemas/model-src-utils.d.ts +6 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +5 -1
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +2 -0
- package/dist/schemas/ocr.js +1 -1
- package/dist/schemas/plugin.d.ts +9 -3
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +8 -0
- package/dist/schemas/registry.js +2 -1
- package/dist/schemas/reload-config.d.ts +6 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/sdcpp-config.d.ts +56 -0
- package/dist/schemas/sdcpp-config.js +33 -15
- package/dist/schemas/text-to-speech.d.ts +61 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +16 -1
- package/dist/schemas/transcription.d.ts +1 -1
- package/dist/schemas/transcription.js +1 -1
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +20 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/surface.d.ts +2 -1
- package/dist/surface.js +2 -1
- package/dist/utils/addon-responses.d.ts +2 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/tts-stats.d.ts +1 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +53 -45
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit → model-fit}/kv-cache.js +0 -0
- /package/dist/{resources/model-fit → model-fit}/platform.js +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
package/dist/logging/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { getLogger, getEngineLogger, getAppLogger, createStreamLogger, setGlobalLogLevel, setGlobalConsoleOutput, registerLogger, unregisterLogger } from '
|
|
2
|
-
export type { Logger, LoggerOptions, LogTransport } from '
|
|
3
|
-
export { RAG_NAMESPACE, LOG_ID, ALL_LOG_ID, LOG_NAMESPACE, type AddonNamespace } from '
|
|
4
|
-
export { registerAddonLogger, unregisterAddonLogger, createAddonLoggerCallback, clearAllAddonLoggers } from '
|
|
5
|
-
export { summarizeRequest } from '
|
|
1
|
+
export { getLogger, getEngineLogger, getAppLogger, createStreamLogger, setGlobalLogLevel, setGlobalConsoleOutput, registerLogger, unregisterLogger } from '../logging/logger.js';
|
|
2
|
+
export type { Logger, LoggerOptions, LogTransport } from '../logging/types.js';
|
|
3
|
+
export { RAG_NAMESPACE, LOG_ID, ALL_LOG_ID, LOG_NAMESPACE, type AddonNamespace } from '../logging/namespaces.js';
|
|
4
|
+
export { registerAddonLogger, unregisterAddonLogger, createAddonLoggerCallback, clearAllAddonLoggers } from '../logging/addon.js';
|
|
5
|
+
export { summarizeRequest } from '../logging/utils.js';
|
package/dist/logging/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { getLogger, getEngineLogger, getAppLogger, createStreamLogger, setGlobalLogLevel, setGlobalConsoleOutput, registerLogger, unregisterLogger } from '
|
|
2
|
-
export { RAG_NAMESPACE, LOG_ID, ALL_LOG_ID, LOG_NAMESPACE } from '
|
|
3
|
-
export { registerAddonLogger, unregisterAddonLogger, createAddonLoggerCallback, clearAllAddonLoggers } from '
|
|
4
|
-
export { summarizeRequest } from '
|
|
1
|
+
export { getLogger, getEngineLogger, getAppLogger, createStreamLogger, setGlobalLogLevel, setGlobalConsoleOutput, registerLogger, unregisterLogger } from '../logging/logger.js';
|
|
2
|
+
export { RAG_NAMESPACE, LOG_ID, ALL_LOG_ID, LOG_NAMESPACE } from '../logging/namespaces.js';
|
|
3
|
+
export { registerAddonLogger, unregisterAddonLogger, createAddonLoggerCallback, clearAllAddonLoggers } from '../logging/addon.js';
|
|
4
|
+
export { summarizeRequest } from '../logging/utils.js';
|
package/dist/logging/logger.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type LogLevel } from '@qvac/logging';
|
|
2
|
-
import type { Logger, LoggerOptions } from '
|
|
2
|
+
import type { Logger, LoggerOptions } from '../logging/types.js';
|
|
3
3
|
export declare function registerLogger(logger: Logger): void;
|
|
4
4
|
export declare function unregisterLogger(logger: Logger): void;
|
|
5
5
|
export declare function setGlobalLogLevel(level: LogLevel): void;
|
package/dist/logging/logger.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import QvacLogger from '@qvac/logging';
|
|
2
|
-
import { LOG_LEVELS } from '@qvac/logging/constants
|
|
2
|
+
import { LOG_LEVELS } from '@qvac/logging/constants';
|
|
3
3
|
import { sendLogToStreams } from '../runtime/logging-stream-registry.js';
|
|
4
|
-
import { isLevelEnabled, formatArg } from '
|
|
5
|
-
import { LOG_ID, LOG_NAMESPACE } from '
|
|
4
|
+
import { isLevelEnabled, formatArg } from '../logging/utils.js';
|
|
5
|
+
import { LOG_ID, LOG_NAMESPACE } from '../logging/namespaces.js';
|
|
6
6
|
// A single instance is shared across every evaluation of this module via
|
|
7
7
|
// globalThis so that log level and console toggles reach all loggers even if
|
|
8
8
|
// the module is evaluated more than once.
|
package/dist/logging/utils.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { LEVEL_PRIORITIES, LOG_LEVELS } from '@qvac/logging/constants
|
|
1
|
+
import { LEVEL_PRIORITIES, LOG_LEVELS } from '@qvac/logging/constants';
|
|
2
2
|
import stringify from 'fast-safe-stringify';
|
|
3
3
|
export function isLevelEnabled(messageLevel, currentLevel) {
|
|
4
4
|
// "off" suppresses every emission (console, streams, and transports alike),
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { AssessModelFitResult, ModelFitBasis, ModelFitEstimateTarget, ModelFitExecution, NativeProbeFit } from '
|
|
2
|
-
import type { SystemResources } from '
|
|
3
|
-
import type { ModelResourceProfile } from '
|
|
4
|
-
import type { ModelFitPlatform } from '
|
|
1
|
+
import type { AssessModelFitResult, ModelFitBasis, ModelFitEstimateTarget, ModelFitExecution, NativeProbeFit } from '../schemas/assess-model-fit.js';
|
|
2
|
+
import type { SystemResources } from '../schemas/system-resources.js';
|
|
3
|
+
import type { ModelResourceProfile } from '../schemas/model-resource-profile.js';
|
|
4
|
+
import type { ModelFitPlatform } from '../model-fit/platform.js';
|
|
5
5
|
/** Resolves a checksum to its catalog resource profile. */
|
|
6
6
|
export type ProfileResolver = (sha256Checksum: string) => ModelResourceProfile | undefined;
|
|
7
7
|
export interface AssessModelFitOptions {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { getModelResourceProfile } from '
|
|
2
|
-
import { computeFloor, FLOOR_VERSION } from '
|
|
1
|
+
import { getModelResourceProfile } from '../models/registry/resource-profiles.js';
|
|
2
|
+
import { computeFloor, FLOOR_VERSION } from '../model-fit/floor.js';
|
|
3
3
|
const GIB = 1024 * 1024 * 1024;
|
|
4
4
|
const MOBILE_PLATFORMS = ['android-arm64', 'ios-arm64'];
|
|
5
5
|
// Every allocation, GPU included, is system RAM: the system budget always bounds the load.
|
|
@@ -331,7 +331,7 @@ function allocatesFromSystemMemory(gpu) {
|
|
|
331
331
|
* APU also exposes for its carve-out — a Ryzen 5000U reported `dedicated` with
|
|
332
332
|
* over a gigabyte of "VRAM". Vulkan calls the same device INTEGRATED_GPU, but
|
|
333
333
|
* that is not in the collector, so an AMD GPU on linux cannot be placed and
|
|
334
|
-
* assesses as `unknown
|
|
334
|
+
* assesses as `unknown`.
|
|
335
335
|
*/
|
|
336
336
|
function integratedIsIndistinguishable(gpu, platform) {
|
|
337
337
|
if (!platform.startsWith('linux'))
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { NativeProbeFit } from '
|
|
2
|
-
import type { FitProbeResult } from '
|
|
1
|
+
import type { NativeProbeFit } from '../schemas/index.js';
|
|
2
|
+
import type { FitProbeResult } from '../model-fit/run/engine.js';
|
|
3
3
|
export declare function classifyFit(probe: FitProbeResult, provenance: {
|
|
4
4
|
basis: 'native-probe';
|
|
5
5
|
estimatorVersion: string;
|
|
@@ -17,15 +17,15 @@ function verdictFor(status) {
|
|
|
17
17
|
return 'unknown';
|
|
18
18
|
}
|
|
19
19
|
/**
|
|
20
|
-
* The diffusion fitter reports a placement
|
|
21
|
-
*
|
|
22
|
-
*
|
|
20
|
+
* The diffusion fitter reports a placement alongside its reason. `changed`
|
|
21
|
+
* means it reached a fitting arrangement only by moving modules between
|
|
22
|
+
* backends, which is why the status is already `does-not-fit`.
|
|
23
23
|
*/
|
|
24
24
|
function diffusionReason(result) {
|
|
25
25
|
if (result.status === 'fits')
|
|
26
26
|
return 'fits';
|
|
27
27
|
if (result.status === 'error')
|
|
28
|
-
return
|
|
28
|
+
return result.reason;
|
|
29
29
|
return result.changed ? `does-not-fit-as-configured (${result.backend})` : 'does-not-fit';
|
|
30
30
|
}
|
|
31
31
|
function reasonFor(probe) {
|
package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { QVACBlobBinding } from '@qvac/registry-client';
|
|
3
|
-
import type { Logger } from '
|
|
3
|
+
import type { Logger } from '../../logging/types.js';
|
|
4
4
|
/**
|
|
5
5
|
* A weightless description of an artifact, served by the registry beside the
|
|
6
6
|
* weights: for a GGUF, the tensor list and the settings with the tokenizer
|
package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js}
RENAMED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import fs, { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
3
|
import { AbortController } from 'bare-abort-controller';
|
|
4
|
-
import { getCacheDir } from '
|
|
5
|
-
import { calculateFileChecksum } from '
|
|
6
|
-
import { getEngineLogger } from '
|
|
4
|
+
import { getCacheDir } from '../../utils/cache/paths.js';
|
|
5
|
+
import { calculateFileChecksum } from '../../utils/checksum.js';
|
|
6
|
+
import { getEngineLogger } from '../../logging/index.js';
|
|
7
7
|
/**
|
|
8
8
|
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
9
9
|
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
@@ -20,12 +20,12 @@ import { getEngineLogger } from '../../../logging/index.js';
|
|
|
20
20
|
*/
|
|
21
21
|
export const FIT_STUB_BUDGET_MS = 40_000;
|
|
22
22
|
async function defaultGetEntry(registryPath, registrySource) {
|
|
23
|
-
const { getRegistryClient } = await import('
|
|
23
|
+
const { getRegistryClient } = await import('../../runtime/registry-client.js');
|
|
24
24
|
const client = await getRegistryClient();
|
|
25
25
|
return client.getModel(registryPath, registrySource);
|
|
26
26
|
}
|
|
27
27
|
async function defaultDownloadBlob(binding, outputFile, signal) {
|
|
28
|
-
const { getRegistryClient } = await import('
|
|
28
|
+
const { getRegistryClient } = await import('../../runtime/registry-client.js');
|
|
29
29
|
const client = await getRegistryClient();
|
|
30
30
|
return client.downloadBlob(binding, {
|
|
31
31
|
outputFile,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { type LoadDescription } from '
|
|
2
|
-
import type { FitStubOptions, FitStubUnavailableReason } from '
|
|
3
|
-
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '
|
|
1
|
+
import { type LoadDescription } from '../../model-fit/description/resolve-load.js';
|
|
2
|
+
import type { FitStubOptions, FitStubUnavailableReason } from '../../model-fit/description/fetch.js';
|
|
3
|
+
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../../model-fit/load-check.js';
|
|
4
4
|
export type LoadFitOutcome = {
|
|
5
5
|
status: 'projected';
|
|
6
6
|
fit: AdvisoryFitOutcome;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { resolveLoadFromStubs } from '
|
|
2
|
-
import { runAdvisoryFitCheck } from '
|
|
1
|
+
import { resolveLoadFromStubs } from '../../model-fit/description/resolve-load.js';
|
|
2
|
+
import { runAdvisoryFitCheck } from '../../model-fit/load-check.js';
|
|
3
3
|
/**
|
|
4
4
|
* Projects the load a caller describes, before its weights exist locally.
|
|
5
5
|
*
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { TtsFitWorkload } from '
|
|
2
|
-
import { type CanonicalModelType, type ModelSrcInput, type TranscriptionFitWorkload } from '
|
|
3
|
-
import { type FitStubOptions, type FitStubUnavailableReason } from '
|
|
1
|
+
import type { TtsFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
import { type CanonicalModelType, type ModelSrcInput, type TranscriptionFitWorkload } from '../../schemas/index.js';
|
|
3
|
+
import { type FitStubOptions, type FitStubUnavailableReason } from '../../model-fit/description/fetch.js';
|
|
4
4
|
export interface LoadDescription {
|
|
5
5
|
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
6
6
|
ttsWorkload?: TtsFitWorkload | undefined;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import fs from 'bare-fs';
|
|
2
|
-
import { ModelType } from '
|
|
3
|
-
import { getPlugin } from '
|
|
4
|
-
import { getRuntimeContext } from '
|
|
5
|
-
import { formatZodError } from '
|
|
6
|
-
import { getModelByPath } from '
|
|
7
|
-
import { fetchFitStub, fetchFitStubSet, removeStub } from '
|
|
2
|
+
import { ModelType } from '../../schemas/index.js';
|
|
3
|
+
import { getPlugin } from '../../plugins/registry.js';
|
|
4
|
+
import { getRuntimeContext } from '../../runtime/state.js';
|
|
5
|
+
import { formatZodError } from '../../utils/zod-error.js';
|
|
6
|
+
import { getModelByPath } from '../../models/registry/index.js';
|
|
7
|
+
import { fetchFitStub, fetchFitStubSet, removeStub } from '../../model-fit/description/fetch.js';
|
|
8
8
|
function srcString(modelSrc) {
|
|
9
9
|
return typeof modelSrc === 'string' ? modelSrc : modelSrc.src;
|
|
10
10
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ModelFitWorkload } from '
|
|
2
|
-
import type { ModelResourceProfile } from '
|
|
1
|
+
import type { ModelFitWorkload } from '../schemas/assess-model-fit.js';
|
|
2
|
+
import type { ModelResourceProfile } from '../schemas/model-resource-profile.js';
|
|
3
3
|
export declare const FLOOR_VERSION = "floor-v1";
|
|
4
4
|
export interface FloorInput {
|
|
5
5
|
profile: ModelResourceProfile;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheFloorBytes, narrowestKvElementBytes } from '
|
|
1
|
+
import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheFloorBytes, narrowestKvElementBytes } from '../model-fit/kv-cache.js';
|
|
2
2
|
export const FLOOR_VERSION = 'floor-v1';
|
|
3
3
|
/**
|
|
4
4
|
* Engines whose load builds a KV cache the file sizes exactly, so the floor can
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { GgufFacts } from '
|
|
1
|
+
import type { GgufFacts } from '../schemas/model-resource-profile.js';
|
|
2
2
|
export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
|
|
3
3
|
/**
|
|
4
4
|
* The narrowest KV-cache element width the engine can default to for this
|
package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts}
RENAMED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import type { Logger } from '
|
|
3
|
-
import type { TtsFitWorkload } from '
|
|
4
|
-
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict, TranscriptionFitWorkload } from '
|
|
5
|
-
import { runFit as runFitDefault } from '
|
|
2
|
+
import type { Logger } from '../logging/types.js';
|
|
3
|
+
import type { TtsFitWorkload } from '../schemas/assess-model-fit.js';
|
|
4
|
+
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict, TranscriptionFitWorkload } from '../schemas/index.js';
|
|
5
|
+
import { runFit as runFitDefault } from '../model-fit/run/run.js';
|
|
6
6
|
/**
|
|
7
7
|
* `fit` and `does-not-fit` are projections of the load the SDK is about to run.
|
|
8
8
|
* The real loader neither consumes nor verifies the fitted plan.
|
|
@@ -29,8 +29,6 @@ export interface AdvisoryFitInput {
|
|
|
29
29
|
*/
|
|
30
30
|
export interface AdvisoryFitOptions {
|
|
31
31
|
signal?: AbortSignal;
|
|
32
|
-
mobile?: boolean;
|
|
33
|
-
timeoutMs?: number;
|
|
34
32
|
runFit?: typeof runFitDefault;
|
|
35
33
|
logger?: Logger;
|
|
36
34
|
availableSystemBytes?: () => Promise<number | undefined>;
|
|
@@ -46,9 +44,9 @@ export interface AdvisoryFitOptions {
|
|
|
46
44
|
* Runs the advisory fit check for a load that is about to start, against the
|
|
47
45
|
* fitter belonging to the engine that will run it.
|
|
48
46
|
*
|
|
49
|
-
* Fail-open by construction: an unsupported shape, a
|
|
50
|
-
*
|
|
51
|
-
* and the caller proceeds with the ordinary load path.
|
|
52
|
-
* throws and never rejects.
|
|
47
|
+
* Fail-open by construction: an unsupported shape, a fitter that crashed,
|
|
48
|
+
* threw or did not return in time, and an unexpected internal error all
|
|
49
|
+
* resolve to `unknown` and the caller proceeds with the ordinary load path.
|
|
50
|
+
* This function never throws and never rejects.
|
|
53
51
|
*/
|
|
54
52
|
export declare function runAdvisoryFitCheck(input: AdvisoryFitInput, options?: AdvisoryFitOptions): Promise<AdvisoryFitOutcome>;
|
|
@@ -1,13 +1,7 @@
|
|
|
1
|
-
import { getEngineLogger } from '
|
|
2
|
-
import { classifyFit } from '
|
|
3
|
-
import { createFitRequest } from '
|
|
4
|
-
import { runFit as runFitDefault } from '
|
|
5
|
-
/**
|
|
6
|
-
* Shorter than the supervisor's own 60s default: this check sits in front of a
|
|
7
|
-
* real load, so a wedged child must not hold the load for a full minute. Every
|
|
8
|
-
* expiry is `unknown` and the load continues.
|
|
9
|
-
*/
|
|
10
|
-
const ADVISORY_FIT_TIMEOUT_MS = 30_000;
|
|
1
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
2
|
+
import { classifyFit } from '../model-fit/classify.js';
|
|
3
|
+
import { createFitRequest } from '../model-fit/request/create.js';
|
|
4
|
+
import { runFit as runFitDefault } from '../model-fit/run/run.js';
|
|
11
5
|
/**
|
|
12
6
|
* The base every engine's fitter withholds by default. Made explicit here
|
|
13
7
|
* because the resident-model reserve is added to it: setting a margin at all
|
|
@@ -24,17 +18,6 @@ const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
|
|
|
24
18
|
*/
|
|
25
19
|
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v2';
|
|
26
20
|
const BYTES_PER_MIB = 1024 * 1024;
|
|
27
|
-
/**
|
|
28
|
-
* The mobile runtime flag is imported lazily: that module reaches Bare-only
|
|
29
|
-
* bindings, and resolving it eagerly would make this orchestration untestable
|
|
30
|
-
* outside a Bare runtime.
|
|
31
|
-
*/
|
|
32
|
-
async function resolveMobile(explicit) {
|
|
33
|
-
if (explicit !== undefined)
|
|
34
|
-
return explicit;
|
|
35
|
-
const { isMobile } = await import('../../../runtime/state.js');
|
|
36
|
-
return isMobile();
|
|
37
|
-
}
|
|
38
21
|
/**
|
|
39
22
|
* Sums the on-disk weight sizes of every model currently registered in this
|
|
40
23
|
* worker. The fallback for a host whose memory sample is unavailable, where
|
|
@@ -46,7 +29,7 @@ async function resolveMobile(explicit) {
|
|
|
46
29
|
*/
|
|
47
30
|
async function defaultResidentModelBytes() {
|
|
48
31
|
const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
|
|
49
|
-
import('
|
|
32
|
+
import('../runtime/model-registry.js'),
|
|
50
33
|
import('bare-fs')
|
|
51
34
|
]);
|
|
52
35
|
let bytes = 0;
|
|
@@ -84,9 +67,9 @@ function mib(bytes) {
|
|
|
84
67
|
*/
|
|
85
68
|
async function defaultAvailableSystemBytes() {
|
|
86
69
|
const [{ getResourceCollector }, { resolveBasis, resolveBudget }, { detectPlatform }] = await Promise.all([
|
|
87
|
-
import('
|
|
88
|
-
import('../assess.js'),
|
|
89
|
-
import('../platform.js')
|
|
70
|
+
import('../resources/instance.js'),
|
|
71
|
+
import('../model-fit/assess.js'),
|
|
72
|
+
import('../model-fit/platform.js')
|
|
90
73
|
]);
|
|
91
74
|
const collector = getResourceCollector();
|
|
92
75
|
if (!collector)
|
|
@@ -104,11 +87,7 @@ async function defaultAvailableSystemBytes() {
|
|
|
104
87
|
* is system RAM, which is what `boundBySystemMemory` decides.
|
|
105
88
|
*/
|
|
106
89
|
async function deviceBytesCountAgainstSystem() {
|
|
107
|
-
const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([
|
|
108
|
-
import('../../instance.js'),
|
|
109
|
-
import('../assess.js'),
|
|
110
|
-
import('../platform.js')
|
|
111
|
-
]);
|
|
90
|
+
const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([import('../resources/instance.js'), import('../model-fit/assess.js'), import('../model-fit/platform.js')]);
|
|
112
91
|
const collector = getResourceCollector();
|
|
113
92
|
// Unknown counts them, which errs toward refusing.
|
|
114
93
|
if (!collector)
|
|
@@ -192,16 +171,15 @@ function report(logger, input, outcome) {
|
|
|
192
171
|
* Runs the advisory fit check for a load that is about to start, against the
|
|
193
172
|
* fitter belonging to the engine that will run it.
|
|
194
173
|
*
|
|
195
|
-
* Fail-open by construction: an unsupported shape, a
|
|
196
|
-
*
|
|
197
|
-
* and the caller proceeds with the ordinary load path.
|
|
198
|
-
* throws and never rejects.
|
|
174
|
+
* Fail-open by construction: an unsupported shape, a fitter that crashed,
|
|
175
|
+
* threw or did not return in time, and an unexpected internal error all
|
|
176
|
+
* resolve to `unknown` and the caller proceeds with the ordinary load path.
|
|
177
|
+
* This function never throws and never rejects.
|
|
199
178
|
*/
|
|
200
179
|
export async function runAdvisoryFitCheck(input, options = {}) {
|
|
201
180
|
let logger = options.logger;
|
|
202
181
|
try {
|
|
203
182
|
logger ??= getEngineLogger();
|
|
204
|
-
const mobile = await resolveMobile(options.mobile);
|
|
205
183
|
const availableBytes = await (options.availableSystemBytes ?? defaultAvailableSystemBytes)();
|
|
206
184
|
const residentBytes = (await (options.residentModelBytes ?? defaultResidentModelBytes)()) +
|
|
207
185
|
(options.extraResidentBytes ?? 0);
|
|
@@ -223,11 +201,7 @@ export async function runAdvisoryFitCheck(input, options = {}) {
|
|
|
223
201
|
report(logger, input, outcome);
|
|
224
202
|
return outcome;
|
|
225
203
|
}
|
|
226
|
-
const result = await (options.runFit ?? runFitDefault)(plan.probe, {
|
|
227
|
-
mobile,
|
|
228
|
-
timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
|
|
229
|
-
...(options.signal !== undefined && { signal: options.signal })
|
|
230
|
-
});
|
|
204
|
+
const result = await (options.runFit ?? runFitDefault)(plan.probe, options.signal !== undefined ? { signal: options.signal } : {});
|
|
231
205
|
const outcome = result.status === 'completed'
|
|
232
206
|
? withMachineBudget(classifyFit(result.probe, PROVENANCE), availableBytes, await (options.countsDeviceRows ?? deviceBytesCountAgainstSystem)())
|
|
233
207
|
: unknown(result.reason, result.message);
|
|
@@ -4,3 +4,11 @@
|
|
|
4
4
|
* against.
|
|
5
5
|
*/
|
|
6
6
|
export type ModelFitPlatform = 'darwin-arm64' | 'darwin-x64' | 'linux-arm64' | 'linux-x64' | 'win32-x64' | 'win32-arm64' | 'android-arm64' | 'ios-arm64';
|
|
7
|
+
/**
|
|
8
|
+
* Maps the runtime's platform and architecture onto a platform this assessment
|
|
9
|
+
* covers.
|
|
10
|
+
*
|
|
11
|
+
* @returns `undefined` for any pair outside that set, which assesses as
|
|
12
|
+
* `unknown`.
|
|
13
|
+
*/
|
|
14
|
+
export declare function detectPlatform(): ModelFitPlatform | undefined;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { TtsFitWorkload } from '
|
|
2
|
-
import { type CanonicalModelType, type TranscriptionFitWorkload } from '
|
|
3
|
-
import type { FitProbeRequest } from '
|
|
1
|
+
import type { TtsFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
import { type CanonicalModelType, type TranscriptionFitWorkload } from '../../schemas/index.js';
|
|
3
|
+
import type { FitProbeRequest } from '../../model-fit/run/engine.js';
|
|
4
4
|
export type FitRequestPlan = {
|
|
5
5
|
supported: true;
|
|
6
6
|
probe: FitProbeRequest;
|
package/dist/{resources/model-fit/native-probe/create-fit-request.js → model-fit/request/create.js}
RENAMED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { ModelType } from '
|
|
2
|
-
import { createBciFitRequest, createMossTranscribeFitRequest, createParakeetFitRequest, createWhisperFitRequest } from '
|
|
3
|
-
import { createAudiogenFitRequest } from '
|
|
4
|
-
import { createDiffusionFitRequest } from '
|
|
5
|
-
import { createLlamaFitRequest, llamaLoadKindFor } from '
|
|
6
|
-
import { createNmtFitRequest } from '
|
|
7
|
-
import { createOcrFitRequest } from '
|
|
8
|
-
import { createTtsFitRequest } from '
|
|
1
|
+
import { ModelType } from '../../schemas/index.js';
|
|
2
|
+
import { createBciFitRequest, createMossTranscribeFitRequest, createParakeetFitRequest, createWhisperFitRequest } from '../../model-fit/request/engines/asr.js';
|
|
3
|
+
import { createAudiogenFitRequest } from '../../model-fit/request/engines/audiogen.js';
|
|
4
|
+
import { createDiffusionFitRequest } from '../../model-fit/request/engines/diffusion.js';
|
|
5
|
+
import { createLlamaFitRequest, llamaLoadKindFor } from '../../model-fit/request/engines/llama.js';
|
|
6
|
+
import { createNmtFitRequest } from '../../model-fit/request/engines/nmt.js';
|
|
7
|
+
import { createOcrFitRequest } from '../../model-fit/request/engines/ocr.js';
|
|
8
|
+
import { createTtsFitRequest } from '../../model-fit/request/engines/tts.js';
|
|
9
9
|
/**
|
|
10
10
|
* Builds a fit request from the same resolved config and artifacts the real
|
|
11
11
|
* load is about to use, or explains why this load cannot be answered for.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { TranscriptionFitWorkload } from '
|
|
2
|
-
import type { FitRequestPlan } from '
|
|
1
|
+
import type { TranscriptionFitWorkload } from '../../../schemas/index.js';
|
|
2
|
+
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
3
3
|
export interface AsrFitRequestParams {
|
|
4
4
|
modelPath: string;
|
|
5
5
|
modelConfig: unknown;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { gpuLayersFromGate } from '
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/request/gpu.js';
|
|
2
2
|
export function createMossTranscribeFitRequest(params) {
|
|
3
3
|
if (params.transcriptionWorkload === undefined) {
|
|
4
4
|
return {
|
|
@@ -92,6 +92,14 @@ export function createParakeetFitRequest(params) {
|
|
|
92
92
|
modelPath: params.modelPath,
|
|
93
93
|
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
94
94
|
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
95
|
+
// The long-form window bounds the offline encoder graph the projection
|
|
96
|
+
// sizes, so an explicit window (or the one-pass sentinel) reaches the fit.
|
|
97
|
+
...(config.longFormWindowFrames !== undefined && {
|
|
98
|
+
longFormWindowFrames: config.longFormWindowFrames
|
|
99
|
+
}),
|
|
100
|
+
...(config.longFormContextFrames !== undefined && {
|
|
101
|
+
longFormContextFrames: config.longFormContextFrames
|
|
102
|
+
}),
|
|
95
103
|
// Only a streaming load holds a live session, so outside one the chunk
|
|
96
104
|
// cadence describes no allocation the projection should cover.
|
|
97
105
|
...(config.streaming === true &&
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { gpuLayersFromGate } from '
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/request/gpu.js';
|
|
2
2
|
/**
|
|
3
3
|
* `@qvac/audiogen-ggml` is an optional peer dependency, so its value exports
|
|
4
4
|
* are unavailable to a host that ships without it. The engine discriminator is
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { flattenUpscalerKeys, toEsrganAddonConfig } from '../../../plugins/builtin/sdcpp-generation/upscaler-config.js';
|
|
2
|
+
import { splitWorldConfig } from '../../../plugins/builtin/sdcpp-generation/world-config.js';
|
|
3
|
+
/**
|
|
4
|
+
* Frames to project a video load at. The fitter reads one frame when nothing
|
|
5
|
+
* is passed, which projects image generation; `run` generates 33. The frame
|
|
6
|
+
* count is a generation parameter, and this assessment sees load config alone.
|
|
7
|
+
*/
|
|
8
|
+
const DEFAULT_VIDEO_FRAMES = 33;
|
|
9
|
+
export function createDiffusionFitRequest(params) {
|
|
10
|
+
const config = (params.modelConfig ?? {});
|
|
11
|
+
const artifacts = params.artifacts ?? {};
|
|
12
|
+
if (config.mode === 'world') {
|
|
13
|
+
const taehv = artifacts['taehvModelPath'];
|
|
14
|
+
const scene = artifacts['seedScenePath'];
|
|
15
|
+
if (taehv === undefined || scene === undefined) {
|
|
16
|
+
return { supported: false, detail: 'ABot fit requires a resolved decoder and scene pack' };
|
|
17
|
+
}
|
|
18
|
+
const { fitSteps, session } = splitWorldConfig(config.world);
|
|
19
|
+
return {
|
|
20
|
+
supported: true,
|
|
21
|
+
probe: {
|
|
22
|
+
engine: 'diffusion-cpp',
|
|
23
|
+
request: {
|
|
24
|
+
mode: 'world',
|
|
25
|
+
files: { model: params.modelPath, taehv, scene },
|
|
26
|
+
config: session,
|
|
27
|
+
workload: { walkSteps: fitSteps }
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
if (config.mode === 'upscale') {
|
|
33
|
+
return {
|
|
34
|
+
supported: true,
|
|
35
|
+
probe: {
|
|
36
|
+
engine: 'diffusion-cpp',
|
|
37
|
+
request: {
|
|
38
|
+
mode: 'upscale',
|
|
39
|
+
files: { esrgan: params.modelPath },
|
|
40
|
+
config: toEsrganAddonConfig(config),
|
|
41
|
+
workload: { upscaleRepeats: 1 }
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const upscaler = config.mode === 'video' ? undefined : config.upscaler;
|
|
47
|
+
const esrgan = upscaler === undefined ? undefined : artifacts['esrganModelPath'];
|
|
48
|
+
if (upscaler !== undefined && esrgan === undefined) {
|
|
49
|
+
return { supported: false, detail: 'the configured ESRGAN checkpoint is not resolved' };
|
|
50
|
+
}
|
|
51
|
+
const files = {
|
|
52
|
+
model: params.modelPath,
|
|
53
|
+
...(esrgan !== undefined && { esrgan }),
|
|
54
|
+
...(artifacts['clipLModelPath'] !== undefined && { clipL: artifacts['clipLModelPath'] }),
|
|
55
|
+
...(artifacts['clipGModelPath'] !== undefined && { clipG: artifacts['clipGModelPath'] }),
|
|
56
|
+
...(artifacts['t5XxlModelPath'] !== undefined && { t5Xxl: artifacts['t5XxlModelPath'] }),
|
|
57
|
+
...(artifacts['llmModelPath'] !== undefined && { llm: artifacts['llmModelPath'] }),
|
|
58
|
+
...(artifacts['projectionModelPath'] !== undefined && {
|
|
59
|
+
llmVision: artifacts['projectionModelPath']
|
|
60
|
+
}),
|
|
61
|
+
...(artifacts['vaeModelPath'] !== undefined && { vae: artifacts['vaeModelPath'] }),
|
|
62
|
+
...(artifacts['uncondModelPath'] !== undefined && {
|
|
63
|
+
uncondModel: artifacts['uncondModelPath']
|
|
64
|
+
}),
|
|
65
|
+
...(artifacts['clipVisionModelPath'] !== undefined && {
|
|
66
|
+
clipVision: artifacts['clipVisionModelPath']
|
|
67
|
+
}),
|
|
68
|
+
...(artifacts['audioVaeModelPath'] !== undefined && {
|
|
69
|
+
audioVae: artifacts['audioVaeModelPath']
|
|
70
|
+
}),
|
|
71
|
+
...(artifacts['embeddingsConnectorsModelPath'] !== undefined && {
|
|
72
|
+
embeddingsConnectors: artifacts['embeddingsConnectorsModelPath']
|
|
73
|
+
})
|
|
74
|
+
};
|
|
75
|
+
// `mode` and `upscaler` are this SDK's own keys; the engine config has no
|
|
76
|
+
// field for either.
|
|
77
|
+
const { mode: _mode, upscaler: _upscaler, ...engineConfig } = config;
|
|
78
|
+
return {
|
|
79
|
+
supported: true,
|
|
80
|
+
probe: {
|
|
81
|
+
engine: 'diffusion-cpp',
|
|
82
|
+
request: {
|
|
83
|
+
files,
|
|
84
|
+
config: { ...engineConfig, ...flattenUpscalerKeys(upscaler) },
|
|
85
|
+
workload: {
|
|
86
|
+
...(config.vae_tiling !== undefined && { vaeTiling: config.vae_tiling }),
|
|
87
|
+
...(config.mode === 'video' && { videoFrames: DEFAULT_VIDEO_FRAMES }),
|
|
88
|
+
...(upscaler !== undefined && { upscaleRepeats: 1 })
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type CanonicalModelType } from '
|
|
2
|
-
import type { FitRequestPlan } from '
|
|
1
|
+
import { type CanonicalModelType } from '../../../schemas/index.js';
|
|
2
|
+
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
3
3
|
export type LlamaLoadKind = 'completion' | 'embedding';
|
|
4
4
|
export declare function llamaLoadKindFor(modelType: CanonicalModelType): LlamaLoadKind | undefined;
|
|
5
5
|
export interface LlamaFitRequestParams {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { ModelType } from '
|
|
2
|
-
import { transformLlmConfig } from '
|
|
3
|
-
import { transformEmbedConfig } from '
|
|
1
|
+
import { ModelType } from '../../../schemas/index.js';
|
|
2
|
+
import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
|
|
3
|
+
import { transformEmbedConfig } from '../../../plugins/builtin/llamacpp-embedding/transform.js';
|
|
4
4
|
export function llamaLoadKindFor(modelType) {
|
|
5
5
|
if (modelType === ModelType.llamacppCompletion)
|
|
6
6
|
return 'completion';
|
|
@@ -24,13 +24,16 @@ export function createLlamaFitRequest(params) {
|
|
|
24
24
|
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
25
25
|
};
|
|
26
26
|
if (loadKind === 'completion') {
|
|
27
|
-
// The
|
|
28
|
-
//
|
|
29
|
-
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
27
|
+
// The addon's fitter rejects these as unknown arguments, and none changes
|
|
28
|
+
// what it projects: the image limit gates images at inference, and the
|
|
29
|
+
// prompt-cache keys size host-RAM and disk state the fitter does not model.
|
|
30
|
+
const { 'image-max-megapixels': _imageMaxMegapixels, cache_checkpoints: _cacheCheckpoints, cache_checkpoints_max_bytes: _cacheCheckpointsMaxBytes, cache_checkpoint_storage: _cacheCheckpointStorage, cache_checkpoint_dir: _cacheCheckpointDir, cache_ram_mib: _cacheRamMib, ...fitConfig } = modelConfig;
|
|
31
|
+
const config = transformLlmConfig(fitConfig);
|
|
32
|
+
// The fit reads the draft model (DFlash model or external MTP head) like the load does.
|
|
33
|
+
const draftModelPath = params.artifacts?.['draftModelPath'];
|
|
34
|
+
if (draftModelPath)
|
|
35
|
+
config['spec_draft_model'] = draftModelPath;
|
|
36
|
+
const llm = { ...request, config };
|
|
34
37
|
return { supported: true, probe: { engine: 'llm-llamacpp', request: llm } };
|
|
35
38
|
}
|
|
36
39
|
const embed = {
|