@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -1,2 +1,11 @@
|
|
|
1
1
|
import { type RagSearchParams } from '../../schemas/index.js';
|
|
2
|
-
export
|
|
2
|
+
export interface SearchIdentity {
|
|
3
|
+
requestId?: string | undefined;
|
|
4
|
+
sessionId?: string | undefined;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* A search is not a registered RAG operation, so its query is embedded here
|
|
8
|
+
* under the search's own request id and session, then handed to the
|
|
9
|
+
* workspace's embedding function through the stash; see `embedding-scope`.
|
|
10
|
+
*/
|
|
11
|
+
export declare function search(params: RagSearchParams, identity?: SearchIdentity): Promise<import("@qvac/rag").SearchResult[]>;
|
|
@@ -1,16 +1,25 @@
|
|
|
1
1
|
import { getRagInstance, hasRagWorkspaceStorage } from '../../rag/rag-workspace-manager.js';
|
|
2
|
+
import { createWorkspaceEmbeddingFunction, queryEmbeddingText, stashQueryEmbedding, takeQueryEmbedding } from '../../rag/embedding-scope.js';
|
|
2
3
|
import { embed } from '../../plugins/ops/embed.js';
|
|
3
4
|
import { ragSearchParamsSchema } from '../../schemas/index.js';
|
|
4
|
-
|
|
5
|
+
/**
|
|
6
|
+
* A search is not a registered RAG operation, so its query is embedded here
|
|
7
|
+
* under the search's own request id and session, then handed to the
|
|
8
|
+
* workspace's embedding function through the stash; see `embedding-scope`.
|
|
9
|
+
*/
|
|
10
|
+
export async function search(params, identity = {}) {
|
|
5
11
|
const { modelId, query, topK, n, workspace } = ragSearchParamsSchema.parse(params);
|
|
6
12
|
if (!hasRagWorkspaceStorage(workspace)) {
|
|
7
13
|
return [];
|
|
8
14
|
}
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
15
|
+
const rag = await getRagInstance(modelId, createWorkspaceEmbeddingFunction(modelId, workspace), workspace);
|
|
16
|
+
const text = queryEmbeddingText(query);
|
|
17
|
+
const { embedding } = await embed({ modelId, text }, identity.requestId, identity.sessionId);
|
|
18
|
+
stashQueryEmbedding(workspace, text, embedding);
|
|
19
|
+
try {
|
|
20
|
+
return await rag.search(query, { topK, n });
|
|
21
|
+
}
|
|
22
|
+
finally {
|
|
23
|
+
takeQueryEmbedding(workspace, text);
|
|
12
24
|
}
|
|
13
|
-
const rag = await getRagInstance(modelId, embeddingFunction, workspace);
|
|
14
|
-
const results = await rag.search(query, { topK, n });
|
|
15
|
-
return results;
|
|
16
25
|
}
|
|
@@ -1,24 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG operation tracking — workspace-level admission bookkeeping.
|
|
3
|
+
*
|
|
4
|
+
* In 0.11.0 the cancel surface for in-flight RAG operations consolidates
|
|
5
|
+
* onto the request registry (`getRequestRegistry().cancel({ requestId })`).
|
|
6
|
+
* The historical workspace-level pre-emption rule — "starting a new
|
|
7
|
+
* `ingest` / `reindex` / `saveEmbeddings` on a workspace cancels any prior
|
|
8
|
+
* op on the same workspace" — is preserved by the dispatcher in
|
|
9
|
+
* `server/rpc/handlers/rag.ts`: the dispatcher cancels the workspace's
|
|
10
|
+
* prior `requestId` (if any) and then begins a new registry context.
|
|
11
|
+
* Workspace-level admission lives in the dispatcher rather than as a
|
|
12
|
+
* registry policy primitive — it's a dispatch concern, not part of the
|
|
13
|
+
* registry's per-`kind` admission rules.
|
|
14
|
+
*
|
|
15
|
+
* This module owns the small workspace → requestId map that makes that
|
|
16
|
+
* pre-emption decision routable from the dispatcher. The map is module-
|
|
17
|
+
* scoped (one per Bare process) so the dispatcher, the shutdown sweep
|
|
18
|
+
* (`cancelAllRagOperations`), and the workspace-close sweep (which
|
|
19
|
+
* delegates to it via `rag-workspace-manager.ts`) all see the same state.
|
|
20
|
+
*/
|
|
21
|
+
export interface ActiveRagRequest {
|
|
22
|
+
requestId: string;
|
|
23
|
+
sessionId: string;
|
|
24
|
+
}
|
|
1
25
|
export declare function getWorkspaceKey(workspace?: string): string;
|
|
2
26
|
/**
|
|
3
27
|
* Returns the `requestId` of any in-flight RAG operation on the workspace
|
|
4
28
|
* (or `undefined` if none). The dispatcher uses this to decide whether to
|
|
5
29
|
* pre-empt before calling `registry.begin(...)`.
|
|
6
30
|
*/
|
|
7
|
-
export declare function getActiveRagRequest(workspace?: string):
|
|
31
|
+
export declare function getActiveRagRequest(workspace?: string): ActiveRagRequest | undefined;
|
|
8
32
|
/**
|
|
9
33
|
* Records the `requestId` of a freshly-begun RAG operation. Called by the
|
|
10
34
|
* dispatcher in `rag.ts` after `registry.begin(...)` succeeds; paired with
|
|
11
35
|
* `clearActiveRagRequest` via the request scope's deferred cleanup so the
|
|
12
36
|
* map never outlives the request.
|
|
13
37
|
*/
|
|
14
|
-
export declare function setActiveRagRequest(workspace: string | undefined,
|
|
38
|
+
export declare function setActiveRagRequest(workspace: string | undefined, request: ActiveRagRequest): void;
|
|
15
39
|
/**
|
|
16
40
|
* Clears the workspace's mapping iff it still belongs to `requestId`. The
|
|
17
41
|
* conditional guard handles the natural race between two ingest calls on
|
|
18
42
|
* the same workspace: the older context's scope unwind must not stomp the
|
|
19
43
|
* newer context's mapping installed by the pre-emption sequence.
|
|
20
44
|
*/
|
|
21
|
-
export declare function clearActiveRagRequest(workspace: string | undefined,
|
|
45
|
+
export declare function clearActiveRagRequest(workspace: string | undefined, request: ActiveRagRequest): void;
|
|
22
46
|
/**
|
|
23
47
|
* Shutdown / workspace-close sweep. Cancels every tracked RAG request via
|
|
24
48
|
* the registry and clears the workspace map. Idempotent — callers that
|
|
@@ -5,27 +5,7 @@ import { getRequestRegistry } from '../runtime/index.js';
|
|
|
5
5
|
// (unit tests, doc generators) — `rag-workspace-manager` pulls in
|
|
6
6
|
// `bare-fs` / `bare-path` at module load.
|
|
7
7
|
const DEFAULT_WORKSPACE = 'default';
|
|
8
|
-
|
|
9
|
-
* RAG operation tracking — workspace-level admission bookkeeping.
|
|
10
|
-
*
|
|
11
|
-
* In 0.11.0 the cancel surface for in-flight RAG operations consolidates
|
|
12
|
-
* onto the request registry (`getRequestRegistry().cancel({ requestId })`).
|
|
13
|
-
* The historical workspace-level pre-emption rule — "starting a new
|
|
14
|
-
* `ingest` / `reindex` / `saveEmbeddings` on a workspace cancels any prior
|
|
15
|
-
* op on the same workspace" — is preserved by the dispatcher in
|
|
16
|
-
* `server/rpc/handlers/rag.ts`: the dispatcher cancels the workspace's
|
|
17
|
-
* prior `requestId` (if any) and then begins a new registry context.
|
|
18
|
-
* Workspace-level admission lives in the dispatcher rather than as a
|
|
19
|
-
* registry policy primitive — it's a dispatch concern, not part of the
|
|
20
|
-
* registry's per-`kind` admission rules.
|
|
21
|
-
*
|
|
22
|
-
* This module owns the small workspace → requestId map that makes that
|
|
23
|
-
* pre-emption decision routable from the dispatcher. The map is module-
|
|
24
|
-
* scoped (one per Bare process) so the dispatcher, the shutdown sweep
|
|
25
|
-
* (`cancelAllRagOperations`), and the workspace-close sweep (which
|
|
26
|
-
* delegates to it via `rag-workspace-manager.ts`) all see the same state.
|
|
27
|
-
*/
|
|
28
|
-
// workspace key → requestId of the in-flight RAG operation on that workspace.
|
|
8
|
+
// workspace key → identity of the in-flight RAG operation on that workspace.
|
|
29
9
|
const activeRagRequestByWorkspace = new Map();
|
|
30
10
|
export function getWorkspaceKey(workspace) {
|
|
31
11
|
return workspace ?? DEFAULT_WORKSPACE;
|
|
@@ -44,8 +24,11 @@ export function getActiveRagRequest(workspace) {
|
|
|
44
24
|
* `clearActiveRagRequest` via the request scope's deferred cleanup so the
|
|
45
25
|
* map never outlives the request.
|
|
46
26
|
*/
|
|
47
|
-
export function setActiveRagRequest(workspace,
|
|
48
|
-
activeRagRequestByWorkspace.set(getWorkspaceKey(workspace),
|
|
27
|
+
export function setActiveRagRequest(workspace, request) {
|
|
28
|
+
activeRagRequestByWorkspace.set(getWorkspaceKey(workspace), request);
|
|
29
|
+
}
|
|
30
|
+
function sameRequest(a, b) {
|
|
31
|
+
return a !== undefined && a.requestId === b.requestId && a.sessionId === b.sessionId;
|
|
49
32
|
}
|
|
50
33
|
/**
|
|
51
34
|
* Clears the workspace's mapping iff it still belongs to `requestId`. The
|
|
@@ -53,9 +36,9 @@ export function setActiveRagRequest(workspace, requestId) {
|
|
|
53
36
|
* the same workspace: the older context's scope unwind must not stomp the
|
|
54
37
|
* newer context's mapping installed by the pre-emption sequence.
|
|
55
38
|
*/
|
|
56
|
-
export function clearActiveRagRequest(workspace,
|
|
39
|
+
export function clearActiveRagRequest(workspace, request) {
|
|
57
40
|
const key = getWorkspaceKey(workspace);
|
|
58
|
-
if (activeRagRequestByWorkspace.get(key)
|
|
41
|
+
if (sameRequest(activeRagRequestByWorkspace.get(key), request)) {
|
|
59
42
|
activeRagRequestByWorkspace.delete(key);
|
|
60
43
|
}
|
|
61
44
|
}
|
|
@@ -69,8 +52,8 @@ export function cancelAllRagOperations() {
|
|
|
69
52
|
if (activeRagRequestByWorkspace.size === 0)
|
|
70
53
|
return;
|
|
71
54
|
const registry = getRequestRegistry();
|
|
72
|
-
for (const [key,
|
|
73
|
-
registry.cancel({
|
|
55
|
+
for (const [key, request] of activeRagRequestByWorkspace) {
|
|
56
|
+
registry.cancel({ ...request, reason: 'rag-shutdown' });
|
|
74
57
|
activeRagRequestByWorkspace.delete(key);
|
|
75
58
|
}
|
|
76
59
|
}
|
package/dist/registry.js
CHANGED
|
@@ -20,6 +20,7 @@ import { handleModelRegistryList, handleModelRegistrySearch, handleModelRegistry
|
|
|
20
20
|
import { handleSuspend } from './handlers/suspend.js';
|
|
21
21
|
import { handleResume } from './handlers/resume.js';
|
|
22
22
|
import { handleState } from './handlers/state.js';
|
|
23
|
+
import { handleOpenSession, handleCloseSession } from './handlers/session.js';
|
|
23
24
|
// Capability handlers route a request to whichever plugin serves the loaded
|
|
24
25
|
// model's type. Every model capability shares this shape, so one factory each
|
|
25
26
|
// for reply/stream/duplex stands in for a file per capability.
|
|
@@ -51,11 +52,15 @@ export const registry = {
|
|
|
51
52
|
type: 'reply',
|
|
52
53
|
handler: handleHeartbeat
|
|
53
54
|
},
|
|
55
|
+
openSession: { type: 'reply', handler: handleOpenSession },
|
|
56
|
+
closeSession: { type: 'reply', handler: handleCloseSession },
|
|
54
57
|
unloadModel: {
|
|
55
58
|
type: 'reply',
|
|
56
59
|
handler: handleUnloadModel
|
|
57
60
|
},
|
|
58
61
|
embed: { type: 'reply', pluginOp: true, handler: pluginReply('embed') },
|
|
62
|
+
calcTokens: { type: 'reply', pluginOp: true, handler: pluginReply('calcTokens') },
|
|
63
|
+
embedBatch: { type: 'reply', pluginOp: true, handler: pluginReply('embedBatch') },
|
|
59
64
|
decide: { type: 'reply', pluginOp: true, handler: pluginReply('decide') },
|
|
60
65
|
rerank: { type: 'reply', pluginOp: true, handler: pluginReply('rerank') },
|
|
61
66
|
cancel: {
|
|
@@ -9,7 +9,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
9
9
|
value: string;
|
|
10
10
|
provenance: {
|
|
11
11
|
source: string;
|
|
12
|
-
scope?: "
|
|
12
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
13
13
|
};
|
|
14
14
|
} | {
|
|
15
15
|
status: "unavailable";
|
|
@@ -26,7 +26,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
26
26
|
value: string;
|
|
27
27
|
provenance: {
|
|
28
28
|
source: string;
|
|
29
|
-
scope?: "
|
|
29
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
30
30
|
};
|
|
31
31
|
} | {
|
|
32
32
|
status: "unavailable";
|
|
@@ -43,7 +43,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
43
43
|
value: number;
|
|
44
44
|
provenance: {
|
|
45
45
|
source: string;
|
|
46
|
-
scope?: "
|
|
46
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
47
47
|
};
|
|
48
48
|
} | {
|
|
49
49
|
status: "unavailable";
|
|
@@ -60,7 +60,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
60
60
|
value: number;
|
|
61
61
|
provenance: {
|
|
62
62
|
source: string;
|
|
63
|
-
scope?: "
|
|
63
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
64
64
|
};
|
|
65
65
|
} | {
|
|
66
66
|
status: "unavailable";
|
|
@@ -77,7 +77,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
77
77
|
value: number;
|
|
78
78
|
provenance: {
|
|
79
79
|
source: string;
|
|
80
|
-
scope?: "
|
|
80
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
81
81
|
};
|
|
82
82
|
} | {
|
|
83
83
|
status: "unavailable";
|
|
@@ -94,7 +94,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
94
94
|
value: number;
|
|
95
95
|
provenance: {
|
|
96
96
|
source: string;
|
|
97
|
-
scope?: "
|
|
97
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
98
98
|
};
|
|
99
99
|
} | {
|
|
100
100
|
status: "unavailable";
|
|
@@ -111,7 +111,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
111
111
|
value: number;
|
|
112
112
|
provenance: {
|
|
113
113
|
source: string;
|
|
114
|
-
scope?: "
|
|
114
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
115
115
|
};
|
|
116
116
|
} | {
|
|
117
117
|
status: "unavailable";
|
|
@@ -128,7 +128,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
128
128
|
value: number;
|
|
129
129
|
provenance: {
|
|
130
130
|
source: string;
|
|
131
|
-
scope?: "
|
|
131
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
132
132
|
};
|
|
133
133
|
} | {
|
|
134
134
|
status: "unavailable";
|
|
@@ -145,7 +145,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
145
145
|
value: number;
|
|
146
146
|
provenance: {
|
|
147
147
|
source: string;
|
|
148
|
-
scope?: "
|
|
148
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
149
149
|
};
|
|
150
150
|
} | {
|
|
151
151
|
status: "unavailable";
|
|
@@ -160,7 +160,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
160
160
|
};
|
|
161
161
|
provenance: {
|
|
162
162
|
source: string;
|
|
163
|
-
scope?: "
|
|
163
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
164
164
|
};
|
|
165
165
|
} | {
|
|
166
166
|
status: "unavailable";
|
|
@@ -178,7 +178,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
178
178
|
value: number;
|
|
179
179
|
provenance: {
|
|
180
180
|
source: string;
|
|
181
|
-
scope?: "
|
|
181
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
182
182
|
};
|
|
183
183
|
} | {
|
|
184
184
|
status: "unavailable";
|
|
@@ -200,7 +200,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
200
200
|
value: string;
|
|
201
201
|
provenance: {
|
|
202
202
|
source: string;
|
|
203
|
-
scope?: "
|
|
203
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
204
204
|
};
|
|
205
205
|
} | {
|
|
206
206
|
status: "unavailable";
|
|
@@ -217,7 +217,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
217
217
|
value: string;
|
|
218
218
|
provenance: {
|
|
219
219
|
source: string;
|
|
220
|
-
scope?: "
|
|
220
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
221
221
|
};
|
|
222
222
|
} | {
|
|
223
223
|
status: "unavailable";
|
|
@@ -234,7 +234,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
234
234
|
value: number;
|
|
235
235
|
provenance: {
|
|
236
236
|
source: string;
|
|
237
|
-
scope?: "
|
|
237
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
238
238
|
};
|
|
239
239
|
} | {
|
|
240
240
|
status: "unavailable";
|
|
@@ -251,7 +251,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
251
251
|
value: string;
|
|
252
252
|
provenance: {
|
|
253
253
|
source: string;
|
|
254
|
-
scope?: "
|
|
254
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
255
255
|
};
|
|
256
256
|
} | {
|
|
257
257
|
status: "unavailable";
|
|
@@ -268,7 +268,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
268
268
|
value: string;
|
|
269
269
|
provenance: {
|
|
270
270
|
source: string;
|
|
271
|
-
scope?: "
|
|
271
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
272
272
|
};
|
|
273
273
|
} | {
|
|
274
274
|
status: "unavailable";
|
|
@@ -286,7 +286,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
286
286
|
value: boolean;
|
|
287
287
|
provenance: {
|
|
288
288
|
source: string;
|
|
289
|
-
scope?: "
|
|
289
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
290
290
|
};
|
|
291
291
|
} | {
|
|
292
292
|
status: "unavailable";
|
|
@@ -303,7 +303,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
303
303
|
value: boolean;
|
|
304
304
|
provenance: {
|
|
305
305
|
source: string;
|
|
306
|
-
scope?: "
|
|
306
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
307
307
|
};
|
|
308
308
|
} | {
|
|
309
309
|
status: "unavailable";
|
|
@@ -320,7 +320,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
320
320
|
value: boolean;
|
|
321
321
|
provenance: {
|
|
322
322
|
source: string;
|
|
323
|
-
scope?: "
|
|
323
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
324
324
|
};
|
|
325
325
|
} | {
|
|
326
326
|
status: "unavailable";
|
|
@@ -337,7 +337,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
337
337
|
value: boolean;
|
|
338
338
|
provenance: {
|
|
339
339
|
source: string;
|
|
340
|
-
scope?: "
|
|
340
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
341
341
|
};
|
|
342
342
|
} | {
|
|
343
343
|
status: "unavailable";
|
|
@@ -354,7 +354,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
354
354
|
value: boolean;
|
|
355
355
|
provenance: {
|
|
356
356
|
source: string;
|
|
357
|
-
scope?: "
|
|
357
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
358
358
|
};
|
|
359
359
|
} | {
|
|
360
360
|
status: "unavailable";
|
|
@@ -371,7 +371,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
371
371
|
value: boolean;
|
|
372
372
|
provenance: {
|
|
373
373
|
source: string;
|
|
374
|
-
scope?: "
|
|
374
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
375
375
|
};
|
|
376
376
|
} | {
|
|
377
377
|
status: "unavailable";
|
|
@@ -388,7 +388,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
388
388
|
value: boolean;
|
|
389
389
|
provenance: {
|
|
390
390
|
source: string;
|
|
391
|
-
scope?: "
|
|
391
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
392
392
|
};
|
|
393
393
|
} | {
|
|
394
394
|
status: "unavailable";
|
|
@@ -405,7 +405,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
405
405
|
value: boolean;
|
|
406
406
|
provenance: {
|
|
407
407
|
source: string;
|
|
408
|
-
scope?: "
|
|
408
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
409
409
|
};
|
|
410
410
|
} | {
|
|
411
411
|
status: "unavailable";
|
|
@@ -422,7 +422,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
422
422
|
value: boolean;
|
|
423
423
|
provenance: {
|
|
424
424
|
source: string;
|
|
425
|
-
scope?: "
|
|
425
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
426
426
|
};
|
|
427
427
|
} | {
|
|
428
428
|
status: "unavailable";
|
|
@@ -439,7 +439,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
439
439
|
value: boolean;
|
|
440
440
|
provenance: {
|
|
441
441
|
source: string;
|
|
442
|
-
scope?: "
|
|
442
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
443
443
|
};
|
|
444
444
|
} | {
|
|
445
445
|
status: "unavailable";
|
|
@@ -457,7 +457,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
457
457
|
value: boolean;
|
|
458
458
|
provenance: {
|
|
459
459
|
source: string;
|
|
460
|
-
scope?: "
|
|
460
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
461
461
|
};
|
|
462
462
|
} | {
|
|
463
463
|
status: "unavailable";
|
|
@@ -474,7 +474,24 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
474
474
|
value: number;
|
|
475
475
|
provenance: {
|
|
476
476
|
source: string;
|
|
477
|
-
scope?: "
|
|
477
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
478
|
+
};
|
|
479
|
+
} | {
|
|
480
|
+
status: "unavailable";
|
|
481
|
+
reason?: string | undefined;
|
|
482
|
+
} | {
|
|
483
|
+
status: "unverified";
|
|
484
|
+
reason?: string | undefined;
|
|
485
|
+
} | {
|
|
486
|
+
status: "failed";
|
|
487
|
+
reason?: string | undefined;
|
|
488
|
+
};
|
|
489
|
+
dedicatedMemoryTotalBytes: {
|
|
490
|
+
status: "supported";
|
|
491
|
+
value: number;
|
|
492
|
+
provenance: {
|
|
493
|
+
source: string;
|
|
494
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
478
495
|
};
|
|
479
496
|
} | {
|
|
480
497
|
status: "unavailable";
|
|
@@ -489,7 +506,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
489
506
|
}[];
|
|
490
507
|
provenance: {
|
|
491
508
|
source: string;
|
|
492
|
-
scope?: "
|
|
509
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
493
510
|
};
|
|
494
511
|
} | {
|
|
495
512
|
status: "unavailable";
|
|
@@ -523,7 +540,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
523
540
|
value: number;
|
|
524
541
|
provenance: {
|
|
525
542
|
source: string;
|
|
526
|
-
scope?: "
|
|
543
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
527
544
|
};
|
|
528
545
|
} | {
|
|
529
546
|
status: "unavailable";
|
|
@@ -540,7 +557,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
540
557
|
value: number;
|
|
541
558
|
provenance: {
|
|
542
559
|
source: string;
|
|
543
|
-
scope?: "
|
|
560
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
544
561
|
};
|
|
545
562
|
} | {
|
|
546
563
|
status: "unavailable";
|
|
@@ -557,7 +574,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
557
574
|
value: number;
|
|
558
575
|
provenance: {
|
|
559
576
|
source: string;
|
|
560
|
-
scope?: "
|
|
577
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
561
578
|
};
|
|
562
579
|
} | {
|
|
563
580
|
status: "unavailable";
|
|
@@ -574,7 +591,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
574
591
|
value: number;
|
|
575
592
|
provenance: {
|
|
576
593
|
source: string;
|
|
577
|
-
scope?: "
|
|
594
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
578
595
|
};
|
|
579
596
|
} | {
|
|
580
597
|
status: "unavailable";
|
|
@@ -591,7 +608,24 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
591
608
|
value: number;
|
|
592
609
|
provenance: {
|
|
593
610
|
source: string;
|
|
594
|
-
scope?: "
|
|
611
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
612
|
+
};
|
|
613
|
+
} | {
|
|
614
|
+
status: "unavailable";
|
|
615
|
+
reason?: string | undefined;
|
|
616
|
+
} | {
|
|
617
|
+
status: "unverified";
|
|
618
|
+
reason?: string | undefined;
|
|
619
|
+
} | {
|
|
620
|
+
status: "failed";
|
|
621
|
+
reason?: string | undefined;
|
|
622
|
+
};
|
|
623
|
+
dedicatedMemoryUsedBytes: {
|
|
624
|
+
status: "supported";
|
|
625
|
+
value: number;
|
|
626
|
+
provenance: {
|
|
627
|
+
source: string;
|
|
628
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
595
629
|
};
|
|
596
630
|
} | {
|
|
597
631
|
status: "unavailable";
|
|
@@ -608,7 +642,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
608
642
|
value: number;
|
|
609
643
|
provenance: {
|
|
610
644
|
source: string;
|
|
611
|
-
scope?: "
|
|
645
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
612
646
|
};
|
|
613
647
|
} | {
|
|
614
648
|
status: "unavailable";
|
|
@@ -625,7 +659,7 @@ export declare function createSystemResourceCollector(dependencies: ResourceColl
|
|
|
625
659
|
value: number;
|
|
626
660
|
provenance: {
|
|
627
661
|
source: string;
|
|
628
|
-
scope?: "
|
|
662
|
+
scope?: "system" | "process" | "device" | "budget" | "shared-system" | undefined;
|
|
629
663
|
};
|
|
630
664
|
} | {
|
|
631
665
|
status: "unavailable";
|
|
@@ -112,7 +112,10 @@ function normalizeGPUCapabilities(gpu, id, gpuTypes) {
|
|
|
112
112
|
driverVersion: normalizeStringMetric(gpu.driverVersion, gpuDeviceProvenance),
|
|
113
113
|
drivers: normalizeDriverCapabilities(gpu.drivers),
|
|
114
114
|
unifiedMemory: normalizeBooleanMetric(gpu.unifiedMemory, gpuDeviceProvenance),
|
|
115
|
-
memoryTotalBytes: normalizeDeclaredGPUMemory(gpu.memory, gpu.unifiedMemory)
|
|
115
|
+
memoryTotalBytes: normalizeDeclaredGPUMemory(gpu.memory, gpu.unifiedMemory),
|
|
116
|
+
// Trusted even when the rest of the GPU's memory is shared: an integrated
|
|
117
|
+
// GPU's carve-out is reserved for it and the OS does not count it as RAM.
|
|
118
|
+
dedicatedMemoryTotalBytes: normalizeNonNegativeMetric(gpu.dedicatedMemory, gpuDeviceProvenance)
|
|
116
119
|
};
|
|
117
120
|
}
|
|
118
121
|
function failedCPUSample(reason) {
|
|
@@ -133,6 +136,7 @@ function failedGPUSample(id, reason) {
|
|
|
133
136
|
decode: failedMetric(reason),
|
|
134
137
|
memoryUsedBytes: failedMetric(reason),
|
|
135
138
|
memoryTotalBytes: failedMetric(reason),
|
|
139
|
+
dedicatedMemoryUsedBytes: failedMetric(reason),
|
|
136
140
|
powerWatts: failedMetric(reason),
|
|
137
141
|
temperatureCelsius: failedMetric(reason)
|
|
138
142
|
};
|
|
@@ -154,6 +158,9 @@ function normalizeGPUSample(id, usage, device, platform) {
|
|
|
154
158
|
decode: normalizeUtilizationMetric(usage.decode, gpuDeviceProvenance),
|
|
155
159
|
memoryUsedBytes: normalizeSampledGPUMemory(usage.memoryUsed, scope, 'GPU memory usage scope is unverified'),
|
|
156
160
|
memoryTotalBytes: normalizeSampledGPUMemory(usage.memoryTotal, scope, GPU_MEMORY_SCOPE_REASON),
|
|
161
|
+
// A discrete GPU's dedicated usage is its memory usage, so it shares that
|
|
162
|
+
// scope. An integrated GPU's carve-out usage is read device-wide.
|
|
163
|
+
dedicatedMemoryUsedBytes: normalizeSampledGPUMemory(usage.dedicatedMemoryUsed, device?.unifiedMemory === true ? 'device' : scope, 'GPU memory usage scope is unverified'),
|
|
157
164
|
powerWatts: normalizeNonNegativeMetric(usage.power, gpuDeviceProvenance),
|
|
158
165
|
temperatureCelsius: normalizeNonNegativeMetric(usage.temperature, gpuDeviceProvenance)
|
|
159
166
|
};
|
|
@@ -35,6 +35,7 @@ export interface NativeGPUCapabilities {
|
|
|
35
35
|
revision: number;
|
|
36
36
|
unifiedMemory: boolean;
|
|
37
37
|
memory: number | undefined;
|
|
38
|
+
dedicatedMemory: number | undefined;
|
|
38
39
|
}
|
|
39
40
|
export interface NativeGPUUsage {
|
|
40
41
|
compute: number | undefined;
|
|
@@ -42,6 +43,7 @@ export interface NativeGPUUsage {
|
|
|
42
43
|
decode: number | undefined;
|
|
43
44
|
memoryUsed: number | undefined;
|
|
44
45
|
memoryTotal: number | undefined;
|
|
46
|
+
dedicatedMemoryUsed: number | undefined;
|
|
45
47
|
power: number | undefined;
|
|
46
48
|
temperature: number | undefined;
|
|
47
49
|
}
|
package/dist/rpc/inventory.js
CHANGED
|
@@ -3,14 +3,8 @@ import { Buffer } from 'bare-buffer';
|
|
|
3
3
|
import { InferenceCancelledError } from '../errors/index.js';
|
|
4
4
|
import { getEngineLogger } from '../logging/index.js';
|
|
5
5
|
import { throwIfAborted } from './network.js';
|
|
6
|
-
// Fabric RPC_PROTO_*_VERSION:
|
|
7
|
-
// https://github.com/tetherto/qvac-fabric-llm.cpp/blob/v11259.1.0/ggml/include/ggml-rpc.h
|
|
8
6
|
// Command IDs and HELLO layouts, rpc_cmd and rpc_msg_hello_*:
|
|
9
|
-
// https://github.com/tetherto/qvac-fabric-llm.cpp/blob/
|
|
10
|
-
// Native inference needs matching protocol majors, so only servers on the bundled
|
|
11
|
-
// Fabric's protocol are listed. Patch versions are compatible.
|
|
12
|
-
const SUPPORTED_VERSIONS = [{ major: 112, minor: 0 }];
|
|
13
|
-
const SUPPORTED_VERSION_LABEL = SUPPORTED_VERSIONS.map(({ major, minor }) => `${major}.${minor}.x`).join(', ');
|
|
7
|
+
// https://github.com/tetherto/qvac-fabric-llm.cpp/blob/v11739.0.0/ggml/src/ggml-rpc/ggml-rpc.cpp
|
|
14
8
|
const HELLO = 14;
|
|
15
9
|
const HELLO_REQUEST_SIZE = 24;
|
|
16
10
|
const HELLO_RESPONSE_SIZE = 28;
|
|
@@ -55,7 +49,6 @@ export function queryRpcInventory(host, port, timeoutMs, signal, scope) {
|
|
|
55
49
|
url: `${host}:${port}`,
|
|
56
50
|
reason,
|
|
57
51
|
receivedVersion: version,
|
|
58
|
-
supportedVersions: SUPPORTED_VERSION_LABEL,
|
|
59
52
|
receivedHelloBytes: size?.toString() ?? 'unavailable',
|
|
60
53
|
expectedHelloBytes: HELLO_RESPONSE_SIZE
|
|
61
54
|
});
|
|
@@ -102,10 +95,6 @@ export function queryRpcInventory(host, port, timeoutMs, signal, scope) {
|
|
|
102
95
|
}
|
|
103
96
|
if (pending.length < 8 + responseSize)
|
|
104
97
|
return;
|
|
105
|
-
if (command === HELLO &&
|
|
106
|
-
!SUPPORTED_VERSIONS.some(({ major, minor }) => pending[8] === major && pending[9] === minor)) {
|
|
107
|
-
return rejectHello('unsupported protocol version');
|
|
108
|
-
}
|
|
109
98
|
const payload = Buffer.from(pending.subarray(8));
|
|
110
99
|
pending = Buffer.alloc(0);
|
|
111
100
|
if (command === HELLO) {
|
package/dist/runtime/env.d.ts
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
export interface EnvOptions {
|
|
2
|
+
/** Folder that holds `.qvac`. Defaults to the user's home folder. */
|
|
3
|
+
homeDir?: string | undefined;
|
|
4
|
+
}
|
|
1
5
|
/**
|
|
2
6
|
* Initialize the environment. Call once at startup.
|
|
3
7
|
*/
|
|
4
|
-
export declare function initEnv(): void;
|
|
8
|
+
export declare function initEnv(options?: EnvOptions): void;
|
|
5
9
|
/**
|
|
6
10
|
* Get the engine environment. Must call initEnv() first.
|
|
7
11
|
*/
|