@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js}
RENAMED
|
@@ -1,9 +1,61 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
1
53
|
import fs, { promises as fsPromises } from 'bare-fs';
|
|
2
54
|
import path from 'bare-path';
|
|
3
55
|
import { AbortController } from 'bare-abort-controller';
|
|
4
|
-
import { getCacheDir } from '
|
|
5
|
-
import { calculateFileChecksum } from '
|
|
6
|
-
import { getEngineLogger } from '
|
|
56
|
+
import { getCacheDir } from '../../utils/cache/paths.js';
|
|
57
|
+
import { calculateFileChecksum } from '../../utils/checksum.js';
|
|
58
|
+
import { getEngineLogger } from '../../logging/index.js';
|
|
7
59
|
/**
|
|
8
60
|
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
9
61
|
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
@@ -19,19 +71,46 @@ import { getEngineLogger } from '../../../logging/index.js';
|
|
|
19
71
|
* is about to answer is not cut at the moment it would have.
|
|
20
72
|
*/
|
|
21
73
|
export const FIT_STUB_BUDGET_MS = 40_000;
|
|
22
|
-
async function defaultGetEntry(registryPath, registrySource) {
|
|
23
|
-
const {
|
|
24
|
-
|
|
25
|
-
|
|
74
|
+
async function defaultGetEntry(registryPath, registrySource, signal) {
|
|
75
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
76
|
+
try {
|
|
77
|
+
const { acquireRegistryClient } = await import('../../runtime/registry-client.js');
|
|
78
|
+
const lease = __addDisposableResource(env_1, await acquireRegistryClient({ wait: 'query', signal })
|
|
79
|
+
// Awaited here: returning the promise would release the lease before the lookup ends.
|
|
80
|
+
, true);
|
|
81
|
+
// Awaited here: returning the promise would release the lease before the lookup ends.
|
|
82
|
+
return await lease.client.getModel(registryPath, registrySource);
|
|
83
|
+
}
|
|
84
|
+
catch (e_1) {
|
|
85
|
+
env_1.error = e_1;
|
|
86
|
+
env_1.hasError = true;
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
const result_1 = __disposeResources(env_1);
|
|
90
|
+
if (result_1)
|
|
91
|
+
await result_1;
|
|
92
|
+
}
|
|
26
93
|
}
|
|
27
94
|
async function defaultDownloadBlob(binding, outputFile, signal) {
|
|
28
|
-
const {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
95
|
+
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
96
|
+
try {
|
|
97
|
+
const { acquireRegistryClient } = await import('../../runtime/registry-client.js');
|
|
98
|
+
const lease = __addDisposableResource(env_2, await acquireRegistryClient({ wait: 'query', signal }), true);
|
|
99
|
+
return await lease.client.downloadBlob(binding, {
|
|
100
|
+
outputFile,
|
|
101
|
+
maxRetries: 1,
|
|
102
|
+
...(signal !== undefined && { signal })
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
catch (e_2) {
|
|
106
|
+
env_2.error = e_2;
|
|
107
|
+
env_2.hasError = true;
|
|
108
|
+
}
|
|
109
|
+
finally {
|
|
110
|
+
const result_2 = __disposeResources(env_2);
|
|
111
|
+
if (result_2)
|
|
112
|
+
await result_2;
|
|
113
|
+
}
|
|
35
114
|
}
|
|
36
115
|
/** Removes a fetched stub and the directory it was staged in. Never throws. */
|
|
37
116
|
export async function removeStub(stubPath) {
|
|
@@ -101,12 +180,12 @@ async function makeStagingDir(options) {
|
|
|
101
180
|
const created = await fsPromises.mkdtemp(path.join(root, 'stub-'));
|
|
102
181
|
return path.join(root, path.basename(created));
|
|
103
182
|
}
|
|
104
|
-
async function bindingFor(ref, options) {
|
|
183
|
+
async function bindingFor(ref, options, signal) {
|
|
105
184
|
const { registryPath, registrySource } = ref;
|
|
106
185
|
if (!registryPath || !registrySource)
|
|
107
186
|
return 'unresolvable-ref';
|
|
108
187
|
const getEntry = options.getEntry ?? defaultGetEntry;
|
|
109
|
-
const entry = await getEntry(registryPath, registrySource);
|
|
188
|
+
const entry = await getEntry(registryPath, registrySource, signal);
|
|
110
189
|
if (entry === null || entry === undefined)
|
|
111
190
|
return 'not-in-registry';
|
|
112
191
|
const binding = entry.fitBlobBinding;
|
|
@@ -148,7 +227,7 @@ async function stageOne(ref, binding, dest, options, signal) {
|
|
|
148
227
|
async function fetchWithin(ref, options, signal) {
|
|
149
228
|
const logger = options.logger ?? getEngineLogger();
|
|
150
229
|
try {
|
|
151
|
-
const binding = await bindingFor(ref, options);
|
|
230
|
+
const binding = await bindingFor(ref, options, signal);
|
|
152
231
|
if (typeof binding === 'string')
|
|
153
232
|
return unavailable(binding);
|
|
154
233
|
if (signal.aborted)
|
|
@@ -172,7 +251,7 @@ async function fetchSetWithin(set, options, signal) {
|
|
|
172
251
|
let dir;
|
|
173
252
|
let ready = false;
|
|
174
253
|
try {
|
|
175
|
-
const binding = await bindingFor(set.primary, options);
|
|
254
|
+
const binding = await bindingFor(set.primary, options, signal);
|
|
176
255
|
if (typeof binding === 'string')
|
|
177
256
|
return unavailable(binding);
|
|
178
257
|
if (signal.aborted)
|
|
@@ -185,7 +264,7 @@ async function fetchSetWithin(set, options, signal) {
|
|
|
185
264
|
for (const member of set.others) {
|
|
186
265
|
if (signal.aborted)
|
|
187
266
|
break;
|
|
188
|
-
const memberBinding = await bindingFor(member, options);
|
|
267
|
+
const memberBinding = await bindingFor(member, options, signal);
|
|
189
268
|
if (typeof memberBinding === 'string') {
|
|
190
269
|
logger.debug(`companion ${member.targetName} has no fit stub: ${memberBinding}`);
|
|
191
270
|
continue;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { TtsFitWorkload } from '
|
|
2
|
-
import { type CanonicalModelType, type ModelSrcInput, type TranscriptionFitWorkload } from '
|
|
3
|
-
import { type FitStubOptions, type FitStubUnavailableReason } from '
|
|
1
|
+
import type { TtsFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
import { type CanonicalModelType, type ModelSrcInput, type TranscriptionFitWorkload } from '../../schemas/index.js';
|
|
3
|
+
import { type FitStubOptions, type FitStubUnavailableReason } from '../../model-fit/description/fetch.js';
|
|
4
4
|
export interface LoadDescription {
|
|
5
5
|
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
6
6
|
ttsWorkload?: TtsFitWorkload | undefined;
|
|
@@ -21,9 +21,6 @@ export type LoadResolution = ResolvedLoad | {
|
|
|
21
21
|
status: 'no-stub';
|
|
22
22
|
reason: FitStubUnavailableReason;
|
|
23
23
|
message?: string;
|
|
24
|
-
} | {
|
|
25
|
-
status: 'unsupported-load';
|
|
26
|
-
detail: string;
|
|
27
24
|
};
|
|
28
25
|
/**
|
|
29
26
|
* Resolves a load the way `loadModel` would, against the registry's weightless
|
|
@@ -36,5 +33,10 @@ export type LoadResolution = ResolvedLoad | {
|
|
|
36
33
|
*
|
|
37
34
|
* A source already on disk is read directly, since it describes the load more
|
|
38
35
|
* exactly than a stub does. A files-only engine reads only sources on disk.
|
|
36
|
+
*
|
|
37
|
+
* @throws {PluginNotFoundError} When no plugin handles the model type.
|
|
38
|
+
* @throws {PluginLoadConfigValidationFailedError} When the plugin's schema
|
|
39
|
+
* refuses the config.
|
|
40
|
+
* @throws The plugin's own error when its `resolveConfig` refuses the load.
|
|
39
41
|
*/
|
|
40
42
|
export declare function resolveLoadFromStubs(load: LoadDescription, options?: FitStubOptions): Promise<LoadResolution>;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import fs from 'bare-fs';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
2
|
+
import { PluginLoadConfigValidationFailedError, PluginNotFoundError } from '../../errors/index.js';
|
|
3
|
+
import { ModelType } from '../../schemas/index.js';
|
|
4
|
+
import { getPlugin } from '../../plugins/registry.js';
|
|
5
|
+
import { getRuntimeContext } from '../../runtime/state.js';
|
|
6
|
+
import { formatZodError } from '../../utils/zod-error.js';
|
|
7
|
+
import { getModelByPath } from '../../models/registry/index.js';
|
|
8
|
+
import { fetchFitStub, fetchFitStubSet, removeStub } from '../../model-fit/description/fetch.js';
|
|
8
9
|
function srcString(modelSrc) {
|
|
9
10
|
return typeof modelSrc === 'string' ? modelSrc : modelSrc.src;
|
|
10
11
|
}
|
|
@@ -70,15 +71,19 @@ class NoStubError extends Error {
|
|
|
70
71
|
*
|
|
71
72
|
* A source already on disk is read directly, since it describes the load more
|
|
72
73
|
* exactly than a stub does. A files-only engine reads only sources on disk.
|
|
74
|
+
*
|
|
75
|
+
* @throws {PluginNotFoundError} When no plugin handles the model type.
|
|
76
|
+
* @throws {PluginLoadConfigValidationFailedError} When the plugin's schema
|
|
77
|
+
* refuses the config.
|
|
78
|
+
* @throws The plugin's own error when its `resolveConfig` refuses the load.
|
|
73
79
|
*/
|
|
74
80
|
export async function resolveLoadFromStubs(load, options = {}) {
|
|
75
81
|
const plugin = getPlugin(load.modelType);
|
|
76
|
-
if (!plugin)
|
|
77
|
-
|
|
78
|
-
}
|
|
82
|
+
if (!plugin)
|
|
83
|
+
throw new PluginNotFoundError(load.modelType);
|
|
79
84
|
const parsed = plugin.loadConfigSchema.safeParse(load.modelConfig ?? {});
|
|
80
85
|
if (!parsed.success) {
|
|
81
|
-
|
|
86
|
+
throw new PluginLoadConfigValidationFailedError(load.modelType, formatZodError(parsed.error));
|
|
82
87
|
}
|
|
83
88
|
const modelConfig = parsed.data;
|
|
84
89
|
const staged = [];
|
|
@@ -132,15 +137,10 @@ export async function resolveLoadFromStubs(load, options = {}) {
|
|
|
132
137
|
}
|
|
133
138
|
catch (error) {
|
|
134
139
|
await release();
|
|
135
|
-
if (error instanceof NoStubError)
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
// A config the plugin refuses describes a load that would not start either.
|
|
141
|
-
return {
|
|
142
|
-
status: 'unsupported-load',
|
|
143
|
-
detail: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
|
|
144
|
-
};
|
|
140
|
+
if (!(error instanceof NoStubError))
|
|
141
|
+
throw error;
|
|
142
|
+
return error.message === error.reason
|
|
143
|
+
? { status: 'no-stub', reason: error.reason }
|
|
144
|
+
: { status: 'no-stub', reason: error.reason, message: error.message };
|
|
145
145
|
}
|
|
146
146
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import { runFit as runFitDefault } from '../model-fit/run/run.js';
|
|
3
|
+
import type { ModelFitModelResult } from '../schemas/assess-model-fit.js';
|
|
4
|
+
import type { FitLoad } from '../schemas/plugin.js';
|
|
5
|
+
/** The addon's `assessFit` result for one load. */
|
|
6
|
+
export interface LoadFit {
|
|
7
|
+
result: ModelFitModelResult;
|
|
8
|
+
/** Why the status is `unknown`, for the log. */
|
|
9
|
+
reason?: string;
|
|
10
|
+
}
|
|
11
|
+
export interface FitLoadOptions {
|
|
12
|
+
signal?: AbortSignal;
|
|
13
|
+
runFit?: typeof runFitDefault;
|
|
14
|
+
}
|
|
15
|
+
export declare function unknownFit(reason: string): LoadFit;
|
|
16
|
+
/**
|
|
17
|
+
* The fitter's answer for one load, through the plugin of the engine that
|
|
18
|
+
* would run it.
|
|
19
|
+
*
|
|
20
|
+
* @throws The fitter's own error when it rejects the load, such as a model it
|
|
21
|
+
* cannot read or a request it refuses.
|
|
22
|
+
*/
|
|
23
|
+
export declare function fitLoad(modelType: string, load: FitLoad, options?: FitLoadOptions): Promise<LoadFit>;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { runFit as runFitDefault } from '../model-fit/run/run.js';
|
|
2
|
+
import { ModelType } from '../schemas/index.js';
|
|
3
|
+
export function unknownFit(reason) {
|
|
4
|
+
return { result: { status: 'unknown', devices: [], stats: {} }, reason };
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* The fitter's answer for one load, through the plugin of the engine that
|
|
8
|
+
* would run it.
|
|
9
|
+
*
|
|
10
|
+
* @throws The fitter's own error when it rejects the load, such as a model it
|
|
11
|
+
* cannot read or a request it refuses.
|
|
12
|
+
*/
|
|
13
|
+
export async function fitLoad(modelType, load, options = {}) {
|
|
14
|
+
const { getPlugin } = await import('../plugins/registry.js');
|
|
15
|
+
const plugin = getPlugin(modelType);
|
|
16
|
+
if (plugin?.fitRequest === undefined || plugin.assessFit === undefined) {
|
|
17
|
+
return unknownFit(`no fitter for model type ${modelType}`);
|
|
18
|
+
}
|
|
19
|
+
if (load.transcriptionWorkload !== undefined && plugin.modelType !== ModelType.mossTranscribe) {
|
|
20
|
+
return unknownFit('a transcriptionWorkload sizes MOSS transcription only');
|
|
21
|
+
}
|
|
22
|
+
const plan = plugin.fitRequest(load);
|
|
23
|
+
if ('unknown' in plan)
|
|
24
|
+
return unknownFit(plan.unknown);
|
|
25
|
+
const outcome = await (options.runFit ?? runFitDefault)({ addonPackage: plugin.addonPackage, request: plan.request }, options.signal !== undefined ? { signal: options.signal } : {});
|
|
26
|
+
if (outcome.status === 'unknown')
|
|
27
|
+
return unknownFit(`${outcome.reason}: ${outcome.message}`);
|
|
28
|
+
return { result: outcome.fit };
|
|
29
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { Logger } from '../logging/types.js';
|
|
3
|
+
import { type FitLoadOptions } from '../model-fit/fit-load.js';
|
|
4
|
+
import type { CanonicalModelType, ModelFitProbe } from '../schemas/index.js';
|
|
5
|
+
export interface LoadCheckInput {
|
|
6
|
+
modelId: string;
|
|
7
|
+
modelType: CanonicalModelType;
|
|
8
|
+
modelPath: string;
|
|
9
|
+
modelConfig: unknown;
|
|
10
|
+
artifacts?: Record<string, string> | undefined;
|
|
11
|
+
isShardedModel: boolean;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Injection seams. Every field defaults to the real runtime dependency; tests
|
|
15
|
+
* substitute them rather than mocking modules.
|
|
16
|
+
*/
|
|
17
|
+
export interface LoadCheckOptions {
|
|
18
|
+
signal?: AbortSignal;
|
|
19
|
+
logger?: Logger;
|
|
20
|
+
runFit?: FitLoadOptions['runFit'];
|
|
21
|
+
}
|
|
22
|
+
export interface LoadCheck {
|
|
23
|
+
probe: ModelFitProbe;
|
|
24
|
+
/** Why the load does not fit, for the refusal. Present on `does-not-fit`. */
|
|
25
|
+
detail?: string;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Checks a load that is about to start against what is free, with the fitter
|
|
29
|
+
* of the engine that will run it.
|
|
30
|
+
*
|
|
31
|
+
* A fitter that rejects the load is logged and answers `unknown`, so the
|
|
32
|
+
* check never fails a load of its own accord. This function never throws.
|
|
33
|
+
*/
|
|
34
|
+
export declare function runLoadFitCheck(input: LoadCheckInput, options?: LoadCheckOptions): Promise<LoadCheck>;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
2
|
+
import { fitLoad, unknownFit } from '../model-fit/fit-load.js';
|
|
3
|
+
import { describeAccounting, describePool } from '../model-fit/describe.js';
|
|
4
|
+
import { combineFits } from '../model-fit/sum.js';
|
|
5
|
+
/** The pools the load overflows, or the fitter's own refusal when none does. */
|
|
6
|
+
function refusalDetail(combined) {
|
|
7
|
+
const over = combined.pools.filter((pool) => pool.freeBytes !== undefined && pool.usedBytes > pool.freeBytes);
|
|
8
|
+
if (over.length === 0)
|
|
9
|
+
return 'the engine fitter projects it will not fit its devices';
|
|
10
|
+
return `needs ${over.map(describePool).join(', ')}`;
|
|
11
|
+
}
|
|
12
|
+
function report(logger, input, fit, combined, detail) {
|
|
13
|
+
const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
|
|
14
|
+
const accounting = describeAccounting(combined.pools, [
|
|
15
|
+
{ label: input.modelType, devices: fit.result.devices }
|
|
16
|
+
]);
|
|
17
|
+
if (combined.verdict === 'fits') {
|
|
18
|
+
logger.info(`${prefix} projected to fit — ${accounting}`);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
if (combined.verdict === 'does-not-fit') {
|
|
22
|
+
logger.warn(`${prefix} projected not to fit: ${detail} — ${accounting}`);
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
const line = `${prefix} no fit verdict: ${fit.reason ?? 'a device reported no free memory'}`;
|
|
26
|
+
// A model type with no fitter answers this on every load, so it stays at
|
|
27
|
+
// `debug`; anywhere a fitter should have answered, `info` says why it did not.
|
|
28
|
+
if (fit.result.devices.length === 0 && fit.reason?.startsWith('no fitter') === true) {
|
|
29
|
+
logger.debug(line);
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
logger.info(`${line} — ${accounting}`);
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Checks a load that is about to start against what is free, with the fitter
|
|
36
|
+
* of the engine that will run it.
|
|
37
|
+
*
|
|
38
|
+
* A fitter that rejects the load is logged and answers `unknown`, so the
|
|
39
|
+
* check never fails a load of its own accord. This function never throws.
|
|
40
|
+
*/
|
|
41
|
+
export async function runLoadFitCheck(input, options = {}) {
|
|
42
|
+
const logger = options.logger ?? getEngineLogger();
|
|
43
|
+
let fit;
|
|
44
|
+
try {
|
|
45
|
+
fit = await fitLoad(input.modelType, {
|
|
46
|
+
modelPath: input.modelPath,
|
|
47
|
+
modelConfig: input.modelConfig,
|
|
48
|
+
artifacts: input.artifacts,
|
|
49
|
+
isShardedModel: input.isShardedModel
|
|
50
|
+
}, {
|
|
51
|
+
...(options.signal !== undefined && { signal: options.signal }),
|
|
52
|
+
...(options.runFit !== undefined && { runFit: options.runFit })
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
57
|
+
try {
|
|
58
|
+
logger.warn(`[advisory-fit:${input.modelType}:${input.modelId}] fit failed: ${message}`);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// A failing logger must not turn the check into a load failure.
|
|
62
|
+
}
|
|
63
|
+
return { probe: { verdict: 'unknown', pools: [], models: [unknownFit(message).result] } };
|
|
64
|
+
}
|
|
65
|
+
const combined = combineFits([fit.result]);
|
|
66
|
+
const detail = combined.verdict === 'does-not-fit' ? refusalDetail(combined) : undefined;
|
|
67
|
+
try {
|
|
68
|
+
report(logger, input, fit, combined, detail);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// A failing logger must not turn the check into a load failure.
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
probe: { verdict: combined.verdict, pools: combined.pools, models: [fit.result] },
|
|
75
|
+
...(detail !== undefined && { detail })
|
|
76
|
+
};
|
|
77
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ModelFitDevice, ModelFitModelResult } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
/** One device a load uses, as every addon's `assessFit` reports it. */
|
|
3
|
+
export type AddonFitDevice = ModelFitDevice;
|
|
4
|
+
/** The result every addon's `assessFit` returns. */
|
|
5
|
+
export type AddonFit = ModelFitModelResult;
|
|
6
|
+
/** A request for the fitter of the plugin that owns `addonPackage`. */
|
|
7
|
+
export interface FitProbeRequest {
|
|
8
|
+
addonPackage: string;
|
|
9
|
+
request: unknown;
|
|
10
|
+
}
|
|
11
|
+
/** Calls the fitter of a plugin that owns the request's addon. */
|
|
12
|
+
export declare function callEngineFit(probe: FitProbeRequest): Promise<AddonFit>;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Calls the fitter of a plugin that owns the request's addon. */
|
|
2
|
+
export async function callEngineFit(probe) {
|
|
3
|
+
const { getAllPlugins } = await import('../../plugins/registry.js');
|
|
4
|
+
const plugin = getAllPlugins().find((candidate) => candidate.addonPackage === probe.addonPackage && candidate.assessFit !== undefined);
|
|
5
|
+
if (plugin?.assessFit === undefined) {
|
|
6
|
+
throw new TypeError(`no registered plugin exposes assessFit for ${probe.addonPackage}`);
|
|
7
|
+
}
|
|
8
|
+
return plugin.assessFit(probe.request);
|
|
9
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import { callEngineFit } from '../../model-fit/run/engine.js';
|
|
3
|
+
import type { AddonFit, FitProbeRequest } from '../../model-fit/run/engine.js';
|
|
4
|
+
/**
|
|
5
|
+
* Why a run produced no fit. Each one makes the verdict `unknown`, so the
|
|
6
|
+
* caller proceeds with the load it was going to run anyway.
|
|
7
|
+
*/
|
|
8
|
+
export type FitRunUnknownReason = 'cancelled' | 'crashed' | 'marker-error' | 'timeout';
|
|
9
|
+
export type FitRunResult = {
|
|
10
|
+
status: 'completed';
|
|
11
|
+
fit: AddonFit;
|
|
12
|
+
} | {
|
|
13
|
+
status: 'unknown';
|
|
14
|
+
reason: FitRunUnknownReason;
|
|
15
|
+
message: string;
|
|
16
|
+
};
|
|
17
|
+
export interface RunFitOptions {
|
|
18
|
+
signal?: AbortSignal;
|
|
19
|
+
timeoutMs?: number;
|
|
20
|
+
stateDir?: string;
|
|
21
|
+
callFit?: typeof callEngineFit;
|
|
22
|
+
}
|
|
23
|
+
export declare function runningMarkerPath(stateDir: string, probe: FitProbeRequest, pid?: number): string;
|
|
24
|
+
export declare function suspectMarkerPath(stateDir: string, probe: FitProbeRequest): string;
|
|
25
|
+
export declare function crashedMarkerPath(stateDir: string, probe: FitProbeRequest): string;
|
|
26
|
+
/**
|
|
27
|
+
* Runs one engine's fitter in this process.
|
|
28
|
+
*
|
|
29
|
+
* A fitter that aborts takes the process with it, as does a process that quits
|
|
30
|
+
* or is killed mid-fit. Each process marks its fit with a `.running` file named
|
|
31
|
+
* by its pid, so a marker whose process is gone is a fit that never finished,
|
|
32
|
+
* while one whose process is alive is a fit still running there. The first
|
|
33
|
+
* unfinished fit of a request leaves `.suspect` and the fit runs again. A
|
|
34
|
+
* second one in a row leaves `.crashed`, and the request skips the fitter
|
|
35
|
+
* until that marker expires.
|
|
36
|
+
*
|
|
37
|
+
* Fits of one request in this process share one marker, removed by the last.
|
|
38
|
+
* An aborted `signal` answers `cancelled` and a fit past `timeoutMs` answers
|
|
39
|
+
* `timeout`, both at once, while the fitter runs on and releases the marker
|
|
40
|
+
* when it returns. Until a timed-out fitter returns, further fits of its
|
|
41
|
+
* request answer `timeout` without calling it again.
|
|
42
|
+
*
|
|
43
|
+
* A fitter that rejects, for a model it cannot read or a request it refuses,
|
|
44
|
+
* rejects this call with its error.
|
|
45
|
+
*/
|
|
46
|
+
export declare function runFit(probe: FitProbeRequest, options?: RunFitOptions): Promise<FitRunResult>;
|