@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import fs from 'bare-fs';
|
|
2
|
+
import os from 'bare-os';
|
|
3
|
+
import path from 'bare-path';
|
|
4
|
+
import { generateShortHash } from '../../utils/formatting.js';
|
|
5
|
+
import { callEngineFit } from '../../model-fit/run/engine.js';
|
|
6
|
+
function unknown(reason, message) {
|
|
7
|
+
return { status: 'unknown', reason, message };
|
|
8
|
+
}
|
|
9
|
+
function describe(error) {
|
|
10
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
11
|
+
}
|
|
12
|
+
/** How long a marker keeps a request from calling its fitter. */
|
|
13
|
+
const MARKER_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
14
|
+
/** A fit sits in front of a load, so a fitter that never returns must not hold it. */
|
|
15
|
+
const FIT_TIMEOUT_MS = 30_000;
|
|
16
|
+
/** Fits this process has in flight, per request. */
|
|
17
|
+
const inFlight = new Map();
|
|
18
|
+
/** Requests whose fit timed out and has not returned. */
|
|
19
|
+
const hung = new Set();
|
|
20
|
+
function markerKey(probe) {
|
|
21
|
+
return generateShortHash(JSON.stringify(probe));
|
|
22
|
+
}
|
|
23
|
+
export function runningMarkerPath(stateDir, probe, pid = os.pid()) {
|
|
24
|
+
return path.join(stateDir, `${markerKey(probe)}.${pid}.running`);
|
|
25
|
+
}
|
|
26
|
+
export function suspectMarkerPath(stateDir, probe) {
|
|
27
|
+
return path.join(stateDir, `${markerKey(probe)}.suspect`);
|
|
28
|
+
}
|
|
29
|
+
export function crashedMarkerPath(stateDir, probe) {
|
|
30
|
+
return path.join(stateDir, `${markerKey(probe)}.crashed`);
|
|
31
|
+
}
|
|
32
|
+
function exists(file) {
|
|
33
|
+
try {
|
|
34
|
+
fs.accessSync(file);
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function expired(file) {
|
|
42
|
+
try {
|
|
43
|
+
return Date.now() - fs.statSync(file).mtimeMs > MARKER_TTL_MS;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
function remove(file) {
|
|
50
|
+
try {
|
|
51
|
+
fs.unlinkSync(file);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// Marker cleanup must not turn an advisory fit into a load failure.
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
function mark(file) {
|
|
58
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
59
|
+
fs.writeFileSync(file, '');
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* A marker with this process's pid is from an earlier process that had the
|
|
63
|
+
* same pid, since this one holds no fit of the request. `EPERM` means the
|
|
64
|
+
* process exists under another user.
|
|
65
|
+
*/
|
|
66
|
+
function isAlive(pid) {
|
|
67
|
+
if (pid === os.pid())
|
|
68
|
+
return false;
|
|
69
|
+
try {
|
|
70
|
+
os.kill(pid, 0);
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
return error.code === 'EPERM';
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/** `.running` markers of this request whose process is gone or that have expired. */
|
|
78
|
+
function unfinishedFits(stateDir, probe) {
|
|
79
|
+
let names;
|
|
80
|
+
try {
|
|
81
|
+
names = fs.readdirSync(stateDir);
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return [];
|
|
85
|
+
}
|
|
86
|
+
const pattern = new RegExp(`^${markerKey(probe)}\\.(\\d+)\\.running$`);
|
|
87
|
+
return names.flatMap((name) => {
|
|
88
|
+
const pid = pattern.exec(name)?.[1];
|
|
89
|
+
if (pid === undefined)
|
|
90
|
+
return [];
|
|
91
|
+
const file = path.join(stateDir, name);
|
|
92
|
+
return isAlive(Number(pid)) && !expired(file) ? [] : [file];
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
async function resolveStateDir(explicit) {
|
|
96
|
+
if (explicit !== undefined)
|
|
97
|
+
return explicit;
|
|
98
|
+
const { getCacheDir } = await import('../../utils/cache/paths.js');
|
|
99
|
+
return getCacheDir('model-fit');
|
|
100
|
+
}
|
|
101
|
+
async function settle(probe, callFit, stateDir) {
|
|
102
|
+
const key = markerKey(probe);
|
|
103
|
+
try {
|
|
104
|
+
return { status: 'completed', fit: await callFit(probe) };
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
const remaining = (inFlight.get(key) ?? 1) - 1;
|
|
108
|
+
if (remaining === 0) {
|
|
109
|
+
inFlight.delete(key);
|
|
110
|
+
hung.delete(key);
|
|
111
|
+
remove(runningMarkerPath(stateDir, probe));
|
|
112
|
+
remove(suspectMarkerPath(stateDir, probe));
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
inFlight.set(key, remaining);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
function bounded(fit, key, timeoutMs, signal) {
|
|
120
|
+
return new Promise((resolve, reject) => {
|
|
121
|
+
const settled = () => {
|
|
122
|
+
clearTimeout(timer);
|
|
123
|
+
signal?.removeEventListener('abort', onAbort);
|
|
124
|
+
};
|
|
125
|
+
const finish = (result) => {
|
|
126
|
+
settled();
|
|
127
|
+
resolve(result);
|
|
128
|
+
};
|
|
129
|
+
const fail = (error) => {
|
|
130
|
+
settled();
|
|
131
|
+
reject(error);
|
|
132
|
+
};
|
|
133
|
+
const onAbort = () => finish(unknown('cancelled', 'Fit was cancelled'));
|
|
134
|
+
const timer = setTimeout(() => {
|
|
135
|
+
hung.add(key);
|
|
136
|
+
finish(unknown('timeout', `Fit did not return within ${timeoutMs} ms`));
|
|
137
|
+
}, timeoutMs);
|
|
138
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
139
|
+
fit.then(finish, fail);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Runs one engine's fitter in this process.
|
|
144
|
+
*
|
|
145
|
+
* A fitter that aborts takes the process with it, as does a process that quits
|
|
146
|
+
* or is killed mid-fit. Each process marks its fit with a `.running` file named
|
|
147
|
+
* by its pid, so a marker whose process is gone is a fit that never finished,
|
|
148
|
+
* while one whose process is alive is a fit still running there. The first
|
|
149
|
+
* unfinished fit of a request leaves `.suspect` and the fit runs again. A
|
|
150
|
+
* second one in a row leaves `.crashed`, and the request skips the fitter
|
|
151
|
+
* until that marker expires.
|
|
152
|
+
*
|
|
153
|
+
* Fits of one request in this process share one marker, removed by the last.
|
|
154
|
+
* An aborted `signal` answers `cancelled` and a fit past `timeoutMs` answers
|
|
155
|
+
* `timeout`, both at once, while the fitter runs on and releases the marker
|
|
156
|
+
* when it returns. Until a timed-out fitter returns, further fits of its
|
|
157
|
+
* request answer `timeout` without calling it again.
|
|
158
|
+
*
|
|
159
|
+
* A fitter that rejects, for a model it cannot read or a request it refuses,
|
|
160
|
+
* rejects this call with its error.
|
|
161
|
+
*/
|
|
162
|
+
export async function runFit(probe, options = {}) {
|
|
163
|
+
if (options.signal?.aborted === true) {
|
|
164
|
+
return unknown('cancelled', 'Fit was cancelled');
|
|
165
|
+
}
|
|
166
|
+
const key = markerKey(probe);
|
|
167
|
+
if (hung.has(key)) {
|
|
168
|
+
return unknown('timeout', 'An earlier fit of this request has not returned');
|
|
169
|
+
}
|
|
170
|
+
const stateDir = await resolveStateDir(options.stateDir);
|
|
171
|
+
const crashed = crashedMarkerPath(stateDir, probe);
|
|
172
|
+
const suspect = suspectMarkerPath(stateDir, probe);
|
|
173
|
+
const held = inFlight.get(key) ?? 0;
|
|
174
|
+
if (held === 0) {
|
|
175
|
+
for (const marker of [crashed, suspect]) {
|
|
176
|
+
if (exists(marker) && expired(marker))
|
|
177
|
+
remove(marker);
|
|
178
|
+
}
|
|
179
|
+
const unfinished = unfinishedFits(stateDir, probe);
|
|
180
|
+
if (exists(crashed) || (unfinished.length > 0 && exists(suspect))) {
|
|
181
|
+
try {
|
|
182
|
+
// Rewriting an existing marker would restart its TTL on every load.
|
|
183
|
+
if (!exists(crashed))
|
|
184
|
+
mark(crashed);
|
|
185
|
+
remove(suspect);
|
|
186
|
+
unfinished.forEach(remove);
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
// Keep the unfinished markers if `.crashed` could not be written so
|
|
190
|
+
// the next attempt still counts them.
|
|
191
|
+
}
|
|
192
|
+
return unknown('crashed', 'Earlier fits did not finish; treating this model as unknown');
|
|
193
|
+
}
|
|
194
|
+
try {
|
|
195
|
+
if (unfinished.length > 0) {
|
|
196
|
+
mark(suspect);
|
|
197
|
+
unfinished.forEach(remove);
|
|
198
|
+
}
|
|
199
|
+
mark(runningMarkerPath(stateDir, probe));
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
return unknown('marker-error', `Fit marker could not be written: ${describe(error)}`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
inFlight.set(key, held + 1);
|
|
206
|
+
const fit = settle(probe, options.callFit ?? callEngineFit, stateDir);
|
|
207
|
+
return bounded(fit, key, options.timeoutMs ?? FIT_TIMEOUT_MS, options.signal);
|
|
208
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { AddonFitDevice } from '../model-fit/run/engine.js';
|
|
2
|
+
import type { ModelFitPool, ModelFitVerdict } from '../schemas/assess-model-fit.js';
|
|
3
|
+
/**
|
|
4
|
+
* The share of a pool's free memory a set of models may take. The rest absorbs
|
|
5
|
+
* what the machine allocates between the check and the loads.
|
|
6
|
+
*/
|
|
7
|
+
export declare const USABLE_SHARE = 0.9;
|
|
8
|
+
/** The pool of host memory: the host CPU's row and every GPU that shares it. */
|
|
9
|
+
export declare const HOST_POOL = "host";
|
|
10
|
+
export interface ModelAnswer {
|
|
11
|
+
status: ModelFitVerdict;
|
|
12
|
+
devices: readonly AddonFitDevice[];
|
|
13
|
+
}
|
|
14
|
+
export interface CombinedFit {
|
|
15
|
+
verdict: ModelFitVerdict;
|
|
16
|
+
pools: ModelFitPool[];
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Sums what a set of models takes on each memory pool and judges the set
|
|
20
|
+
* against what is free.
|
|
21
|
+
*
|
|
22
|
+
* A device joins the pool of its `id`, or the host pool when `unified`. A
|
|
23
|
+
* pool's free and total memory are the lowest figures reported for it.
|
|
24
|
+
*
|
|
25
|
+
* A single model's verdict is its addon's own, which applies the engine's
|
|
26
|
+
* margin. A set does not fit when any pool takes more than
|
|
27
|
+
* `USABLE_SHARE` of its free memory or any model does not fit alone. Usage
|
|
28
|
+
* known so far is a lower bound, so either holds whatever the unknown models
|
|
29
|
+
* add. Otherwise the set is `unknown` when any model is unknown or any pool has
|
|
30
|
+
* no free figure.
|
|
31
|
+
*/
|
|
32
|
+
export declare function combineFits(models: readonly ModelAnswer[]): CombinedFit;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The share of a pool's free memory a set of models may take. The rest absorbs
|
|
3
|
+
* what the machine allocates between the check and the loads.
|
|
4
|
+
*/
|
|
5
|
+
export const USABLE_SHARE = 0.9;
|
|
6
|
+
/** The pool of host memory: the host CPU's row and every GPU that shares it. */
|
|
7
|
+
export const HOST_POOL = 'host';
|
|
8
|
+
function lowest(figures) {
|
|
9
|
+
return figures.length > 0 ? Math.min(...figures) : undefined;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Sums what a set of models takes on each memory pool and judges the set
|
|
13
|
+
* against what is free.
|
|
14
|
+
*
|
|
15
|
+
* A device joins the pool of its `id`, or the host pool when `unified`. A
|
|
16
|
+
* pool's free and total memory are the lowest figures reported for it.
|
|
17
|
+
*
|
|
18
|
+
* A single model's verdict is its addon's own, which applies the engine's
|
|
19
|
+
* margin. A set does not fit when any pool takes more than
|
|
20
|
+
* `USABLE_SHARE` of its free memory or any model does not fit alone. Usage
|
|
21
|
+
* known so far is a lower bound, so either holds whatever the unknown models
|
|
22
|
+
* add. Otherwise the set is `unknown` when any model is unknown or any pool has
|
|
23
|
+
* no free figure.
|
|
24
|
+
*/
|
|
25
|
+
export function combineFits(models) {
|
|
26
|
+
const totals = new Map();
|
|
27
|
+
for (const model of models) {
|
|
28
|
+
for (const device of model.devices) {
|
|
29
|
+
const key = device.unified ? HOST_POOL : device.id;
|
|
30
|
+
const pool = totals.get(key) ?? { usedBytes: 0, free: [], total: [] };
|
|
31
|
+
pool.usedBytes += device.usedBytes;
|
|
32
|
+
if (device.freeBytes !== undefined)
|
|
33
|
+
pool.free.push(device.freeBytes);
|
|
34
|
+
if (device.totalBytes !== undefined)
|
|
35
|
+
pool.total.push(device.totalBytes);
|
|
36
|
+
totals.set(key, pool);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const pools = [...totals].map(([id, { usedBytes, free, total }]) => {
|
|
40
|
+
const freeBytes = lowest(free);
|
|
41
|
+
const totalBytes = lowest(total);
|
|
42
|
+
return {
|
|
43
|
+
id,
|
|
44
|
+
unified: id === HOST_POOL,
|
|
45
|
+
usedBytes,
|
|
46
|
+
...(freeBytes !== undefined && { freeBytes }),
|
|
47
|
+
...(totalBytes !== undefined && { totalBytes })
|
|
48
|
+
};
|
|
49
|
+
});
|
|
50
|
+
const [only] = models;
|
|
51
|
+
if (models.length === 1 && only !== undefined)
|
|
52
|
+
return { verdict: only.status, pools };
|
|
53
|
+
const over = pools.some((pool) => pool.freeBytes !== undefined && pool.usedBytes > pool.freeBytes * USABLE_SHARE);
|
|
54
|
+
if (over || models.some((model) => model.status === 'does-not-fit')) {
|
|
55
|
+
return { verdict: 'does-not-fit', pools };
|
|
56
|
+
}
|
|
57
|
+
const unmeasured = models.some((model) => model.status === 'unknown') ||
|
|
58
|
+
pools.some((pool) => pool.freeBytes === undefined);
|
|
59
|
+
return { verdict: unmeasured ? 'unknown' : 'fits', pools };
|
|
60
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { models } from '
|
|
1
|
+
import { models } from '../../models/registry/models.js';
|
|
2
2
|
// Re-export all generated models
|
|
3
|
-
export * from '
|
|
3
|
+
export * from '../../models/registry/models.js';
|
|
4
4
|
const bciEmbedder = models.find((model) => model.name === 'BCI_EMBEDDER');
|
|
5
5
|
export const BCI_EMBEDDER = {
|
|
6
6
|
name: 'BCI_EMBEDDER',
|