@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -1,222 +1,77 @@
|
|
|
1
1
|
import { getEngineLogger } from '../logging/index.js';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
/**
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
|
|
12
|
-
/**
|
|
13
|
-
* Names the headroom policy: the fitter withholds
|
|
14
|
-
* `ADVISORY_FIT_BASE_MARGIN_MIB` plus the on-disk bytes of every model resident
|
|
15
|
-
* in this worker, and a `fit` is then judged against the budget
|
|
16
|
-
* `assessModelFit` reports, under that platform's basis and less the
|
|
17
|
-
* `interactive-v1` reserve.
|
|
18
|
-
*/
|
|
19
|
-
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v2';
|
|
20
|
-
const BYTES_PER_MIB = 1024 * 1024;
|
|
21
|
-
/**
|
|
22
|
-
* Sums the on-disk weight sizes of every model currently registered in this
|
|
23
|
-
* worker. The fallback for a host whose memory sample is unavailable, where
|
|
24
|
-
* models this worker loaded are the one part of the machine's usage it can
|
|
25
|
-
* still account for.
|
|
26
|
-
*
|
|
27
|
-
* Advisory and fail-open like everything else here: any failure to stat a file
|
|
28
|
-
* contributes zero rather than an error.
|
|
29
|
-
*/
|
|
30
|
-
async function defaultResidentModelBytes() {
|
|
31
|
-
const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
|
|
32
|
-
import('../runtime/model-registry.js'),
|
|
33
|
-
import('bare-fs')
|
|
34
|
-
]);
|
|
35
|
-
let bytes = 0;
|
|
36
|
-
for (const id of getAllModelIds()) {
|
|
37
|
-
const info = getModelInfo(id);
|
|
38
|
-
if (info === null)
|
|
39
|
-
continue;
|
|
40
|
-
try {
|
|
41
|
-
const stats = (await fsPromises.stat(info.path));
|
|
42
|
-
bytes += stats.size;
|
|
43
|
-
}
|
|
44
|
-
catch {
|
|
45
|
-
// Unreadable path: contribute nothing rather than fail the check.
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
return bytes;
|
|
49
|
-
}
|
|
50
|
-
const PROVENANCE = {
|
|
51
|
-
basis: 'native-probe',
|
|
52
|
-
estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
|
|
53
|
-
};
|
|
54
|
-
function unknown(reason, message) {
|
|
55
|
-
return message === undefined
|
|
56
|
-
? { ...PROVENANCE, verdict: 'unknown', reason }
|
|
57
|
-
: { ...PROVENANCE, verdict: 'unknown', reason, message };
|
|
58
|
-
}
|
|
59
|
-
function mib(bytes) {
|
|
60
|
-
return `${Math.round(bytes / BYTES_PER_MIB)} MiB`;
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* What the operating system would give back, which no backend reports. Metal
|
|
64
|
-
* answers with its working-set allowance minus this process's own allocations,
|
|
65
|
-
* so another application's resident pages read as free, and the host figure
|
|
66
|
-
* counts little beyond wired and compressed memory.
|
|
67
|
-
*/
|
|
68
|
-
async function defaultAvailableSystemBytes() {
|
|
69
|
-
const [{ getResourceCollector }, { resolveBasis, resolveBudget }, { detectPlatform }] = await Promise.all([
|
|
70
|
-
import('../resources/instance.js'),
|
|
71
|
-
import('../model-fit/assess.js'),
|
|
72
|
-
import('../model-fit/platform.js')
|
|
73
|
-
]);
|
|
74
|
-
const collector = getResourceCollector();
|
|
75
|
-
if (!collector)
|
|
76
|
-
return undefined;
|
|
77
|
-
const platform = detectPlatform();
|
|
78
|
-
const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
79
|
-
const budget = resolveBudget(resources, platform, resolveBasis(platform), []);
|
|
80
|
-
if (budget === undefined)
|
|
81
|
-
return undefined;
|
|
82
|
-
return Math.max(0, budget.availableAfterReserveBytes);
|
|
83
|
-
}
|
|
84
|
-
/**
|
|
85
|
-
* Whether the device's allocations come out of the pool the system budget
|
|
86
|
-
* measures. Host bytes always do; device bytes do only where every allocation
|
|
87
|
-
* is system RAM, which is what `boundBySystemMemory` decides.
|
|
88
|
-
*/
|
|
89
|
-
async function deviceBytesCountAgainstSystem() {
|
|
90
|
-
const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([import('../resources/instance.js'), import('../model-fit/assess.js'), import('../model-fit/platform.js')]);
|
|
91
|
-
const collector = getResourceCollector();
|
|
92
|
-
// Unknown counts them, which errs toward refusing.
|
|
93
|
-
if (!collector)
|
|
94
|
-
return true;
|
|
95
|
-
const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
96
|
-
return boundBySystemMemory(resources, detectPlatform());
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* Peak the projection places on the pool the system budget measures. A
|
|
100
|
-
* discrete card's memory is not that pool, so charging its bytes here would
|
|
101
|
-
* refuse a load the machine can hold.
|
|
102
|
-
*/
|
|
103
|
-
function projectedDemandBytes(outcome, countDevice) {
|
|
104
|
-
const projection = outcome.projection;
|
|
105
|
-
if (projection === undefined)
|
|
106
|
-
return undefined;
|
|
107
|
-
if (projection.deviceBytes === undefined && projection.hostBytes === undefined)
|
|
108
|
-
return undefined;
|
|
109
|
-
return (countDevice ? (projection.deviceBytes ?? 0) : 0) + (projection.hostBytes ?? 0);
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* Judges a `fit` against the machine. The fitter decides placement and reports
|
|
113
|
-
* what the load costs; only this side knows what is free, so only this side can
|
|
114
|
-
* refuse on capacity.
|
|
115
|
-
*
|
|
116
|
-
* The margin cannot carry this. It is subtracted from each device's own free,
|
|
117
|
-
* and a device whose total sits below the machine's — Metal caps its allowance
|
|
118
|
-
* near 74% of system memory — would be charged the shortfall twice.
|
|
119
|
-
*
|
|
120
|
-
* A projection with no byte totals leaves the engine's verdict standing.
|
|
121
|
-
*/
|
|
122
|
-
function withMachineBudget(outcome, availableBytes, countDevice) {
|
|
123
|
-
if (outcome.verdict !== 'fit' || availableBytes === undefined)
|
|
124
|
-
return outcome;
|
|
125
|
-
const demand = projectedDemandBytes(outcome, countDevice);
|
|
126
|
-
if (demand === undefined)
|
|
127
|
-
return outcome;
|
|
128
|
-
if (demand <= availableBytes)
|
|
129
|
-
return outcome;
|
|
130
|
-
return {
|
|
131
|
-
...outcome,
|
|
132
|
-
verdict: 'does-not-fit',
|
|
133
|
-
reason: 'exceeds-available-memory',
|
|
134
|
-
message: `needs ${mib(demand)} against a ${mib(availableBytes)} budget`
|
|
135
|
-
};
|
|
136
|
-
}
|
|
137
|
-
/** The device and host totals, for the engines that measure them. */
|
|
138
|
-
function footprint(outcome) {
|
|
139
|
-
const projection = outcome.projection;
|
|
140
|
-
if (projection?.deviceBytes === undefined)
|
|
141
|
-
return '';
|
|
142
|
-
const host = projection.hostBytes === undefined ? '' : `, host ${mib(projection.hostBytes)}`;
|
|
143
|
-
return ` — device ${mib(projection.deviceBytes)}${host}`;
|
|
2
|
+
import { fitLoad, unknownFit } from '../model-fit/fit-load.js';
|
|
3
|
+
import { describeAccounting, describePool } from '../model-fit/describe.js';
|
|
4
|
+
import { combineFits } from '../model-fit/sum.js';
|
|
5
|
+
/** The pools the load overflows, or the fitter's own refusal when none does. */
|
|
6
|
+
function refusalDetail(combined) {
|
|
7
|
+
const over = combined.pools.filter((pool) => pool.freeBytes !== undefined && pool.usedBytes > pool.freeBytes);
|
|
8
|
+
if (over.length === 0)
|
|
9
|
+
return 'the engine fitter projects it will not fit its devices';
|
|
10
|
+
return `needs ${over.map(describePool).join(', ')}`;
|
|
144
11
|
}
|
|
145
|
-
function report(logger, input,
|
|
12
|
+
function report(logger, input, fit, combined, detail) {
|
|
146
13
|
const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
14
|
+
const accounting = describeAccounting(combined.pools, [
|
|
15
|
+
{ label: input.modelType, devices: fit.result.devices }
|
|
16
|
+
]);
|
|
17
|
+
if (combined.verdict === 'fits') {
|
|
18
|
+
logger.info(`${prefix} projected to fit — ${accounting}`);
|
|
152
19
|
return;
|
|
153
20
|
}
|
|
154
|
-
if (
|
|
155
|
-
logger.warn(`${prefix} projected not to fit
|
|
21
|
+
if (combined.verdict === 'does-not-fit') {
|
|
22
|
+
logger.warn(`${prefix} projected not to fit: ${detail} — ${accounting}`);
|
|
156
23
|
return;
|
|
157
24
|
}
|
|
158
|
-
|
|
159
|
-
// type with no fitter
|
|
160
|
-
//
|
|
161
|
-
|
|
162
|
-
// there no evidence" is the most useful thing this check can report.
|
|
163
|
-
const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
|
|
164
|
-
if (outcome.reason === 'unsupported-load') {
|
|
25
|
+
const line = `${prefix} no fit verdict: ${fit.reason ?? 'a device reported no free memory'}`;
|
|
26
|
+
// A model type with no fitter answers this on every load, so it stays at
|
|
27
|
+
// `debug`; anywhere a fitter should have answered, `info` says why it did not.
|
|
28
|
+
if (fit.result.devices.length === 0 && fit.reason?.startsWith('no fitter') === true) {
|
|
165
29
|
logger.debug(line);
|
|
166
30
|
return;
|
|
167
31
|
}
|
|
168
|
-
logger.info(line);
|
|
32
|
+
logger.info(`${line} — ${accounting}`);
|
|
169
33
|
}
|
|
170
34
|
/**
|
|
171
|
-
*
|
|
172
|
-
*
|
|
35
|
+
* Checks a load that is about to start against what is free, with the fitter
|
|
36
|
+
* of the engine that will run it.
|
|
173
37
|
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
* resolve to `unknown` and the caller proceeds with the ordinary load path.
|
|
177
|
-
* This function never throws and never rejects.
|
|
38
|
+
* A fitter that rejects the load is logged and answers `unknown`, so the
|
|
39
|
+
* check never fails a load of its own accord. This function never throws.
|
|
178
40
|
*/
|
|
179
|
-
export async function
|
|
180
|
-
|
|
41
|
+
export async function runLoadFitCheck(input, options = {}) {
|
|
42
|
+
const logger = options.logger ?? getEngineLogger();
|
|
43
|
+
let fit;
|
|
181
44
|
try {
|
|
182
|
-
|
|
183
|
-
const availableBytes = await (options.availableSystemBytes ?? defaultAvailableSystemBytes)();
|
|
184
|
-
const residentBytes = (await (options.residentModelBytes ?? defaultResidentModelBytes)()) +
|
|
185
|
-
(options.extraResidentBytes ?? 0);
|
|
186
|
-
const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
|
|
187
|
-
// Always sent, even with a zero reserve, so the base margin has one source
|
|
188
|
-
// and an engine default never decides it.
|
|
189
|
-
const plan = createFitRequest({
|
|
190
|
-
modelType: input.modelType,
|
|
45
|
+
fit = await fitLoad(input.modelType, {
|
|
191
46
|
modelPath: input.modelPath,
|
|
192
47
|
modelConfig: input.modelConfig,
|
|
193
|
-
transcriptionWorkload: input.transcriptionWorkload,
|
|
194
|
-
ttsWorkload: input.ttsWorkload,
|
|
195
48
|
artifacts: input.artifacts,
|
|
196
|
-
isShardedModel: input.isShardedModel
|
|
197
|
-
|
|
49
|
+
isShardedModel: input.isShardedModel
|
|
50
|
+
}, {
|
|
51
|
+
...(options.signal !== undefined && { signal: options.signal }),
|
|
52
|
+
...(options.runFit !== undefined && { runFit: options.runFit })
|
|
198
53
|
});
|
|
199
|
-
if (!plan.supported) {
|
|
200
|
-
const outcome = unknown('unsupported-load', plan.detail);
|
|
201
|
-
report(logger, input, outcome);
|
|
202
|
-
return outcome;
|
|
203
|
-
}
|
|
204
|
-
const result = await (options.runFit ?? runFitDefault)(plan.probe, options.signal !== undefined ? { signal: options.signal } : {});
|
|
205
|
-
const outcome = result.status === 'completed'
|
|
206
|
-
? withMachineBudget(classifyFit(result.probe, PROVENANCE), availableBytes, await (options.countsDeviceRows ?? deviceBytesCountAgainstSystem)())
|
|
207
|
-
: unknown(result.reason, result.message);
|
|
208
|
-
report(logger, input, outcome);
|
|
209
|
-
return outcome;
|
|
210
54
|
}
|
|
211
55
|
catch (error) {
|
|
212
|
-
const
|
|
56
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
213
57
|
try {
|
|
214
|
-
|
|
215
|
-
report(logger, input, outcome);
|
|
58
|
+
logger.warn(`[advisory-fit:${input.modelType}:${input.modelId}] fit failed: ${message}`);
|
|
216
59
|
}
|
|
217
60
|
catch {
|
|
218
|
-
// A failing logger must not turn
|
|
61
|
+
// A failing logger must not turn the check into a load failure.
|
|
219
62
|
}
|
|
220
|
-
return
|
|
63
|
+
return { probe: { verdict: 'unknown', pools: [], models: [unknownFit(message).result] } };
|
|
221
64
|
}
|
|
65
|
+
const combined = combineFits([fit.result]);
|
|
66
|
+
const detail = combined.verdict === 'does-not-fit' ? refusalDetail(combined) : undefined;
|
|
67
|
+
try {
|
|
68
|
+
report(logger, input, fit, combined, detail);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// A failing logger must not turn the check into a load failure.
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
probe: { verdict: combined.verdict, pools: combined.pools, models: [fit.result] },
|
|
75
|
+
...(detail !== undefined && { detail })
|
|
76
|
+
};
|
|
222
77
|
}
|
|
@@ -1,72 +1,12 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
*/
|
|
14
|
-
export type FitProbeRequest = {
|
|
15
|
-
engine: 'llm-llamacpp';
|
|
16
|
-
request: LlmLlamacpp.FitRequest;
|
|
17
|
-
} | {
|
|
18
|
-
engine: 'embed-llamacpp';
|
|
19
|
-
request: EmbedFitRequest;
|
|
20
|
-
} | {
|
|
21
|
-
engine: 'asr-ggml';
|
|
22
|
-
request: AsrFitRequest;
|
|
23
|
-
} | {
|
|
24
|
-
engine: 'bci-whispercpp';
|
|
25
|
-
request: BciFitRequest;
|
|
26
|
-
} | {
|
|
27
|
-
engine: 'tts-ggml';
|
|
28
|
-
request: TtsFitRequest;
|
|
29
|
-
} | {
|
|
30
|
-
engine: 'audiogen-ggml';
|
|
31
|
-
request: AudiogenFitRequest;
|
|
32
|
-
} | {
|
|
33
|
-
engine: 'diffusion-cpp';
|
|
34
|
-
request: DiffusionFitRequest | EsrganFitRequest | WorldFitRequest;
|
|
35
|
-
} | {
|
|
36
|
-
engine: 'ocr-ggml';
|
|
37
|
-
request: OcrFitRequest;
|
|
38
|
-
} | {
|
|
39
|
-
engine: 'translation-nmtcpp';
|
|
40
|
-
request: TranslationNmtcpp.FitRequest;
|
|
41
|
-
};
|
|
42
|
-
export type FitProbeEngine = FitProbeRequest['engine'];
|
|
43
|
-
export type FitProbeResult = {
|
|
44
|
-
engine: 'llm-llamacpp';
|
|
45
|
-
result: LlmLlamacpp.FitResult;
|
|
46
|
-
} | {
|
|
47
|
-
engine: 'embed-llamacpp';
|
|
48
|
-
result: EmbedFitResult;
|
|
49
|
-
} | {
|
|
50
|
-
engine: 'asr-ggml';
|
|
51
|
-
result: AsrFitResult;
|
|
52
|
-
} | {
|
|
53
|
-
engine: 'bci-whispercpp';
|
|
54
|
-
result: BciFitResult;
|
|
55
|
-
} | {
|
|
56
|
-
engine: 'tts-ggml';
|
|
57
|
-
result: TtsFitResult;
|
|
58
|
-
} | {
|
|
59
|
-
engine: 'audiogen-ggml';
|
|
60
|
-
result: AudiogenFitResult;
|
|
61
|
-
} | {
|
|
62
|
-
engine: 'diffusion-cpp';
|
|
63
|
-
result: DiffusionFitResult;
|
|
64
|
-
} | {
|
|
65
|
-
engine: 'ocr-ggml';
|
|
66
|
-
result: OcrFitResult;
|
|
67
|
-
} | {
|
|
68
|
-
engine: 'translation-nmtcpp';
|
|
69
|
-
result: TranslationNmtcpp.FitResult;
|
|
70
|
-
};
|
|
71
|
-
/** Calls one engine's fitter, taken from the plugin that owns its addon. */
|
|
72
|
-
export declare function callEngineFit(probe: FitProbeRequest): Promise<FitProbeResult>;
|
|
1
|
+
import type { ModelFitDevice, ModelFitModelResult } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
/** One device a load uses, as every addon's `assessFit` reports it. */
|
|
3
|
+
export type AddonFitDevice = ModelFitDevice;
|
|
4
|
+
/** The result every addon's `assessFit` returns. */
|
|
5
|
+
export type AddonFit = ModelFitModelResult;
|
|
6
|
+
/** A request for the fitter of the plugin that owns `addonPackage`. */
|
|
7
|
+
export interface FitProbeRequest {
|
|
8
|
+
addonPackage: string;
|
|
9
|
+
request: unknown;
|
|
10
|
+
}
|
|
11
|
+
/** Calls the fitter of a plugin that owns the request's addon. */
|
|
12
|
+
export declare function callEngineFit(probe: FitProbeRequest): Promise<AddonFit>;
|
|
@@ -1,13 +1,9 @@
|
|
|
1
|
-
/** Calls
|
|
1
|
+
/** Calls the fitter of a plugin that owns the request's addon. */
|
|
2
2
|
export async function callEngineFit(probe) {
|
|
3
3
|
const { getAllPlugins } = await import('../../plugins/registry.js');
|
|
4
|
-
const
|
|
5
|
-
const plugin = getAllPlugins().find((candidate) => candidate.addonPackage === addonPackage);
|
|
4
|
+
const plugin = getAllPlugins().find((candidate) => candidate.addonPackage === probe.addonPackage && candidate.assessFit !== undefined);
|
|
6
5
|
if (plugin?.assessFit === undefined) {
|
|
7
|
-
throw new TypeError(`no registered plugin exposes assessFit for ${addonPackage}`);
|
|
6
|
+
throw new TypeError(`no registered plugin exposes assessFit for ${probe.addonPackage}`);
|
|
8
7
|
}
|
|
9
|
-
return
|
|
10
|
-
engine: probe.engine,
|
|
11
|
-
result: await plugin.assessFit(probe.request)
|
|
12
|
-
};
|
|
8
|
+
return plugin.assessFit(probe.request);
|
|
13
9
|
}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import { callEngineFit } from '../../model-fit/run/engine.js';
|
|
3
|
-
import type {
|
|
3
|
+
import type { AddonFit, FitProbeRequest } from '../../model-fit/run/engine.js';
|
|
4
4
|
/**
|
|
5
|
-
* Why a run produced no
|
|
6
|
-
*
|
|
5
|
+
* Why a run produced no fit. Each one makes the verdict `unknown`, so the
|
|
6
|
+
* caller proceeds with the load it was going to run anyway.
|
|
7
7
|
*/
|
|
8
|
-
export type FitRunUnknownReason = 'cancelled' | 'crashed' | '
|
|
8
|
+
export type FitRunUnknownReason = 'cancelled' | 'crashed' | 'marker-error' | 'timeout';
|
|
9
9
|
export type FitRunResult = {
|
|
10
10
|
status: 'completed';
|
|
11
|
-
|
|
11
|
+
fit: AddonFit;
|
|
12
12
|
} | {
|
|
13
13
|
status: 'unknown';
|
|
14
14
|
reason: FitRunUnknownReason;
|
|
@@ -31,13 +31,16 @@ export declare function crashedMarkerPath(stateDir: string, probe: FitProbeReque
|
|
|
31
31
|
* by its pid, so a marker whose process is gone is a fit that never finished,
|
|
32
32
|
* while one whose process is alive is a fit still running there. The first
|
|
33
33
|
* unfinished fit of a request leaves `.suspect` and the fit runs again. A
|
|
34
|
-
* second one in a row leaves `.crashed`, and the request skips
|
|
35
|
-
* that marker expires.
|
|
34
|
+
* second one in a row leaves `.crashed`, and the request skips the fitter
|
|
35
|
+
* until that marker expires.
|
|
36
36
|
*
|
|
37
37
|
* Fits of one request in this process share one marker, removed by the last.
|
|
38
38
|
* An aborted `signal` answers `cancelled` and a fit past `timeoutMs` answers
|
|
39
39
|
* `timeout`, both at once, while the fitter runs on and releases the marker
|
|
40
40
|
* when it returns. Until a timed-out fitter returns, further fits of its
|
|
41
41
|
* request answer `timeout` without calling it again.
|
|
42
|
+
*
|
|
43
|
+
* A fitter that rejects, for a model it cannot read or a request it refuses,
|
|
44
|
+
* rejects this call with its error.
|
|
42
45
|
*/
|
|
43
46
|
export declare function runFit(probe: FitProbeRequest, options?: RunFitOptions): Promise<FitRunResult>;
|
|
@@ -9,7 +9,7 @@ function unknown(reason, message) {
|
|
|
9
9
|
function describe(error) {
|
|
10
10
|
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
11
11
|
}
|
|
12
|
-
/** How long a marker keeps a
|
|
12
|
+
/** How long a marker keeps a request from calling its fitter. */
|
|
13
13
|
const MARKER_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
14
14
|
/** A fit sits in front of a load, so a fitter that never returns must not hold it. */
|
|
15
15
|
const FIT_TIMEOUT_MS = 30_000;
|
|
@@ -17,13 +17,8 @@ const FIT_TIMEOUT_MS = 30_000;
|
|
|
17
17
|
const inFlight = new Map();
|
|
18
18
|
/** Requests whose fit timed out and has not returned. */
|
|
19
19
|
const hung = new Set();
|
|
20
|
-
/**
|
|
21
|
-
* `marginBytes` is excluded: it carries the bytes of every model resident at
|
|
22
|
-
* the time, so it differs between two attempts at the same load.
|
|
23
|
-
*/
|
|
24
20
|
function markerKey(probe) {
|
|
25
|
-
|
|
26
|
-
return generateShortHash(JSON.stringify({ engine: probe.engine, request }));
|
|
21
|
+
return generateShortHash(JSON.stringify(probe));
|
|
27
22
|
}
|
|
28
23
|
export function runningMarkerPath(stateDir, probe, pid = os.pid()) {
|
|
29
24
|
return path.join(stateDir, `${markerKey(probe)}.${pid}.running`);
|
|
@@ -106,10 +101,7 @@ async function resolveStateDir(explicit) {
|
|
|
106
101
|
async function settle(probe, callFit, stateDir) {
|
|
107
102
|
const key = markerKey(probe);
|
|
108
103
|
try {
|
|
109
|
-
return { status: 'completed',
|
|
110
|
-
}
|
|
111
|
-
catch (error) {
|
|
112
|
-
return unknown('invocation-error', describe(error));
|
|
104
|
+
return { status: 'completed', fit: await callFit(probe) };
|
|
113
105
|
}
|
|
114
106
|
finally {
|
|
115
107
|
const remaining = (inFlight.get(key) ?? 1) - 1;
|
|
@@ -125,19 +117,26 @@ async function settle(probe, callFit, stateDir) {
|
|
|
125
117
|
}
|
|
126
118
|
}
|
|
127
119
|
function bounded(fit, key, timeoutMs, signal) {
|
|
128
|
-
return new Promise((resolve) => {
|
|
129
|
-
const
|
|
120
|
+
return new Promise((resolve, reject) => {
|
|
121
|
+
const settled = () => {
|
|
130
122
|
clearTimeout(timer);
|
|
131
123
|
signal?.removeEventListener('abort', onAbort);
|
|
124
|
+
};
|
|
125
|
+
const finish = (result) => {
|
|
126
|
+
settled();
|
|
132
127
|
resolve(result);
|
|
133
128
|
};
|
|
129
|
+
const fail = (error) => {
|
|
130
|
+
settled();
|
|
131
|
+
reject(error);
|
|
132
|
+
};
|
|
134
133
|
const onAbort = () => finish(unknown('cancelled', 'Fit was cancelled'));
|
|
135
134
|
const timer = setTimeout(() => {
|
|
136
135
|
hung.add(key);
|
|
137
136
|
finish(unknown('timeout', `Fit did not return within ${timeoutMs} ms`));
|
|
138
137
|
}, timeoutMs);
|
|
139
138
|
signal?.addEventListener('abort', onAbort, { once: true });
|
|
140
|
-
|
|
139
|
+
fit.then(finish, fail);
|
|
141
140
|
});
|
|
142
141
|
}
|
|
143
142
|
/**
|
|
@@ -148,14 +147,17 @@ function bounded(fit, key, timeoutMs, signal) {
|
|
|
148
147
|
* by its pid, so a marker whose process is gone is a fit that never finished,
|
|
149
148
|
* while one whose process is alive is a fit still running there. The first
|
|
150
149
|
* unfinished fit of a request leaves `.suspect` and the fit runs again. A
|
|
151
|
-
* second one in a row leaves `.crashed`, and the request skips
|
|
152
|
-
* that marker expires.
|
|
150
|
+
* second one in a row leaves `.crashed`, and the request skips the fitter
|
|
151
|
+
* until that marker expires.
|
|
153
152
|
*
|
|
154
153
|
* Fits of one request in this process share one marker, removed by the last.
|
|
155
154
|
* An aborted `signal` answers `cancelled` and a fit past `timeoutMs` answers
|
|
156
155
|
* `timeout`, both at once, while the fitter runs on and releases the marker
|
|
157
156
|
* when it returns. Until a timed-out fitter returns, further fits of its
|
|
158
157
|
* request answer `timeout` without calling it again.
|
|
158
|
+
*
|
|
159
|
+
* A fitter that rejects, for a model it cannot read or a request it refuses,
|
|
160
|
+
* rejects this call with its error.
|
|
159
161
|
*/
|
|
160
162
|
export async function runFit(probe, options = {}) {
|
|
161
163
|
if (options.signal?.aborted === true) {
|
|
@@ -177,7 +179,9 @@ export async function runFit(probe, options = {}) {
|
|
|
177
179
|
const unfinished = unfinishedFits(stateDir, probe);
|
|
178
180
|
if (exists(crashed) || (unfinished.length > 0 && exists(suspect))) {
|
|
179
181
|
try {
|
|
180
|
-
|
|
182
|
+
// Rewriting an existing marker would restart its TTL on every load.
|
|
183
|
+
if (!exists(crashed))
|
|
184
|
+
mark(crashed);
|
|
181
185
|
remove(suspect);
|
|
182
186
|
unfinished.forEach(remove);
|
|
183
187
|
}
|
|
@@ -195,7 +199,7 @@ export async function runFit(probe, options = {}) {
|
|
|
195
199
|
mark(runningMarkerPath(stateDir, probe));
|
|
196
200
|
}
|
|
197
201
|
catch (error) {
|
|
198
|
-
return unknown('
|
|
202
|
+
return unknown('marker-error', `Fit marker could not be written: ${describe(error)}`);
|
|
199
203
|
}
|
|
200
204
|
}
|
|
201
205
|
inFlight.set(key, held + 1);
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { AddonFitDevice } from '../model-fit/run/engine.js';
|
|
2
|
+
import type { ModelFitPool, ModelFitVerdict } from '../schemas/assess-model-fit.js';
|
|
3
|
+
/**
|
|
4
|
+
* The share of a pool's free memory a set of models may take. The rest absorbs
|
|
5
|
+
* what the machine allocates between the check and the loads.
|
|
6
|
+
*/
|
|
7
|
+
export declare const USABLE_SHARE = 0.9;
|
|
8
|
+
/** The pool of host memory: the host CPU's row and every GPU that shares it. */
|
|
9
|
+
export declare const HOST_POOL = "host";
|
|
10
|
+
export interface ModelAnswer {
|
|
11
|
+
status: ModelFitVerdict;
|
|
12
|
+
devices: readonly AddonFitDevice[];
|
|
13
|
+
}
|
|
14
|
+
export interface CombinedFit {
|
|
15
|
+
verdict: ModelFitVerdict;
|
|
16
|
+
pools: ModelFitPool[];
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Sums what a set of models takes on each memory pool and judges the set
|
|
20
|
+
* against what is free.
|
|
21
|
+
*
|
|
22
|
+
* A device joins the pool of its `id`, or the host pool when `unified`. A
|
|
23
|
+
* pool's free and total memory are the lowest figures reported for it.
|
|
24
|
+
*
|
|
25
|
+
* A single model's verdict is its addon's own, which applies the engine's
|
|
26
|
+
* margin. A set does not fit when any pool takes more than
|
|
27
|
+
* `USABLE_SHARE` of its free memory or any model does not fit alone. Usage
|
|
28
|
+
* known so far is a lower bound, so either holds whatever the unknown models
|
|
29
|
+
* add. Otherwise the set is `unknown` when any model is unknown or any pool has
|
|
30
|
+
* no free figure.
|
|
31
|
+
*/
|
|
32
|
+
export declare function combineFits(models: readonly ModelAnswer[]): CombinedFit;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The share of a pool's free memory a set of models may take. The rest absorbs
|
|
3
|
+
* what the machine allocates between the check and the loads.
|
|
4
|
+
*/
|
|
5
|
+
export const USABLE_SHARE = 0.9;
|
|
6
|
+
/** The pool of host memory: the host CPU's row and every GPU that shares it. */
|
|
7
|
+
export const HOST_POOL = 'host';
|
|
8
|
+
function lowest(figures) {
|
|
9
|
+
return figures.length > 0 ? Math.min(...figures) : undefined;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Sums what a set of models takes on each memory pool and judges the set
|
|
13
|
+
* against what is free.
|
|
14
|
+
*
|
|
15
|
+
* A device joins the pool of its `id`, or the host pool when `unified`. A
|
|
16
|
+
* pool's free and total memory are the lowest figures reported for it.
|
|
17
|
+
*
|
|
18
|
+
* A single model's verdict is its addon's own, which applies the engine's
|
|
19
|
+
* margin. A set does not fit when any pool takes more than
|
|
20
|
+
* `USABLE_SHARE` of its free memory or any model does not fit alone. Usage
|
|
21
|
+
* known so far is a lower bound, so either holds whatever the unknown models
|
|
22
|
+
* add. Otherwise the set is `unknown` when any model is unknown or any pool has
|
|
23
|
+
* no free figure.
|
|
24
|
+
*/
|
|
25
|
+
export function combineFits(models) {
|
|
26
|
+
const totals = new Map();
|
|
27
|
+
for (const model of models) {
|
|
28
|
+
for (const device of model.devices) {
|
|
29
|
+
const key = device.unified ? HOST_POOL : device.id;
|
|
30
|
+
const pool = totals.get(key) ?? { usedBytes: 0, free: [], total: [] };
|
|
31
|
+
pool.usedBytes += device.usedBytes;
|
|
32
|
+
if (device.freeBytes !== undefined)
|
|
33
|
+
pool.free.push(device.freeBytes);
|
|
34
|
+
if (device.totalBytes !== undefined)
|
|
35
|
+
pool.total.push(device.totalBytes);
|
|
36
|
+
totals.set(key, pool);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const pools = [...totals].map(([id, { usedBytes, free, total }]) => {
|
|
40
|
+
const freeBytes = lowest(free);
|
|
41
|
+
const totalBytes = lowest(total);
|
|
42
|
+
return {
|
|
43
|
+
id,
|
|
44
|
+
unified: id === HOST_POOL,
|
|
45
|
+
usedBytes,
|
|
46
|
+
...(freeBytes !== undefined && { freeBytes }),
|
|
47
|
+
...(totalBytes !== undefined && { totalBytes })
|
|
48
|
+
};
|
|
49
|
+
});
|
|
50
|
+
const [only] = models;
|
|
51
|
+
if (models.length === 1 && only !== undefined)
|
|
52
|
+
return { verdict: only.status, pools };
|
|
53
|
+
const over = pools.some((pool) => pool.freeBytes !== undefined && pool.usedBytes > pool.freeBytes * USABLE_SHARE);
|
|
54
|
+
if (over || models.some((model) => model.status === 'does-not-fit')) {
|
|
55
|
+
return { verdict: 'does-not-fit', pools };
|
|
56
|
+
}
|
|
57
|
+
const unmeasured = models.some((model) => model.status === 'unknown') ||
|
|
58
|
+
pools.some((pool) => pool.freeBytes === undefined);
|
|
59
|
+
return { verdict: unmeasured ? 'unknown' : 'fits', pools };
|
|
60
|
+
}
|