@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -1,118 +1,476 @@
|
|
|
1
|
+
import fs from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import fsx from 'fs-native-extensions';
|
|
1
4
|
import { QVACRegistryClient } from '@qvac/registry-client';
|
|
2
5
|
import { getCacheDir } from '../utils/cache/paths.js';
|
|
3
|
-
import {
|
|
6
|
+
import { getLifecycleState, registerCorestore, unregisterCorestore } from '../runtime/runtime-lifecycle.js';
|
|
4
7
|
import { getEngineLogger } from '../logging/index.js';
|
|
5
8
|
import { DEFAULT_REGISTRY_CORE_KEY } from '../constants/index.js';
|
|
9
|
+
import { DownloadCancelledError, ModelRegistryBusyError } from '../errors/index.js';
|
|
6
10
|
const logger = getEngineLogger();
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
let
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
11
|
+
const DEFAULT_TIMINGS = {
|
|
12
|
+
graceMs: 5_000,
|
|
13
|
+
pollMs: 250,
|
|
14
|
+
queryWaitMs: 60_000,
|
|
15
|
+
statusWaitMs: 2_000,
|
|
16
|
+
legacyWaitMs: 30_000
|
|
17
|
+
};
|
|
18
|
+
let timings = { ...DEFAULT_TIMINGS };
|
|
19
|
+
/** Test seam: shorten the timings. Called without arguments, restores the defaults. */
|
|
20
|
+
export function setRegistryClientTimingsForTest(next = {}) {
|
|
21
|
+
timings = { ...DEFAULT_TIMINGS, ...next };
|
|
22
|
+
}
|
|
23
|
+
let store = null;
|
|
24
|
+
let refs = 0;
|
|
25
|
+
// Bumped on every close, so a lease from an earlier store cannot release a later one.
|
|
26
|
+
let generation = 0;
|
|
27
|
+
let idleSince = 0;
|
|
28
|
+
// Set while another process waits: new leases wait until the store has been handed over.
|
|
29
|
+
let yielding = false;
|
|
30
|
+
// After handing the store over, the time before this process tries to take it back.
|
|
31
|
+
let handoffUntil = 0;
|
|
32
|
+
let watchTimer = null;
|
|
33
|
+
let opener = null;
|
|
34
|
+
let wakeOpener = null;
|
|
35
|
+
let closing = null;
|
|
36
|
+
let shuttingDown = false;
|
|
37
|
+
const pending = new Set();
|
|
38
|
+
function storeDir() {
|
|
39
|
+
return getCacheDir(`registry-corestore/${DEFAULT_REGISTRY_CORE_KEY}`);
|
|
40
|
+
}
|
|
41
|
+
function markerPath(kind) {
|
|
42
|
+
return path.join(getCacheDir('registry-corestore'), `${DEFAULT_REGISTRY_CORE_KEY}.${kind}`);
|
|
43
|
+
}
|
|
44
|
+
function closeFd(fd) {
|
|
45
|
+
try {
|
|
46
|
+
fs.closeSync(fd);
|
|
47
|
+
}
|
|
48
|
+
catch { }
|
|
49
|
+
}
|
|
50
|
+
function lockedFd(fd) {
|
|
51
|
+
return {
|
|
52
|
+
release() {
|
|
53
|
+
// Unlock before closing: Windows may release a closed handle's lock late.
|
|
54
|
+
try {
|
|
55
|
+
fsx.unlock(fd);
|
|
56
|
+
}
|
|
57
|
+
catch { }
|
|
58
|
+
closeFd(fd);
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
// `held` while another open file description has a conflicting lock;
|
|
63
|
+
// `unavailable` when the marker cannot be opened or locked on this filesystem.
|
|
64
|
+
function tryMarker(kind, shared) {
|
|
65
|
+
let fd;
|
|
66
|
+
try {
|
|
67
|
+
// Not 'a': Windows refuses to lock a handle opened for append only.
|
|
68
|
+
fd = fs.openSync(markerPath(kind), 'a+');
|
|
28
69
|
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
logger.debug(`Registry ${kind} marker unavailable`, { error });
|
|
72
|
+
return 'unavailable';
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
if (fsx.tryLock(fd, { shared }))
|
|
76
|
+
return lockedFd(fd);
|
|
77
|
+
closeFd(fd);
|
|
78
|
+
return 'held';
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
closeFd(fd);
|
|
82
|
+
logger.debug(`Registry ${kind} marker cannot be locked`, { error });
|
|
83
|
+
return 'unavailable';
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
function isMarkerHeld(kind) {
|
|
87
|
+
const marker = tryMarker(kind, false);
|
|
88
|
+
if (marker === 'held')
|
|
89
|
+
return true;
|
|
90
|
+
if (marker === 'unavailable')
|
|
91
|
+
return null;
|
|
92
|
+
marker.release();
|
|
29
93
|
return false;
|
|
30
94
|
}
|
|
31
|
-
|
|
32
|
-
|
|
95
|
+
// Whether another process has the store open. A shared lock answers without
|
|
96
|
+
// blocking the holder's reads on Windows, where locks are mandatory. Anything
|
|
97
|
+
// inconclusive reports `free` and lets the open attempt decide.
|
|
98
|
+
function probeStore() {
|
|
99
|
+
let fd;
|
|
100
|
+
try {
|
|
101
|
+
fd = fs.openSync(path.join(storeDir(), 'CORESTORE'), 'r+');
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return 'free';
|
|
105
|
+
}
|
|
106
|
+
try {
|
|
107
|
+
if (!fsx.tryLock(fd, { shared: true }))
|
|
108
|
+
return 'busy';
|
|
109
|
+
fsx.unlock(fd);
|
|
110
|
+
return 'free';
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return 'free';
|
|
114
|
+
}
|
|
115
|
+
finally {
|
|
116
|
+
closeFd(fd);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
function isFdLockError(error) {
|
|
120
|
+
return error instanceof Error && error.message.includes('File descriptor could not be locked');
|
|
33
121
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
122
|
+
function sleep(ms) {
|
|
123
|
+
return new Promise((resolve) => {
|
|
124
|
+
const timer = setTimeout(done, ms);
|
|
125
|
+
function done() {
|
|
126
|
+
clearTimeout(timer);
|
|
127
|
+
wakeOpener = null;
|
|
128
|
+
resolve();
|
|
129
|
+
}
|
|
130
|
+
wakeOpener = done;
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
function lifecycleActive() {
|
|
134
|
+
return getLifecycleState() === 'active';
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Lease the registry client, opening the store if needed and waiting while
|
|
138
|
+
* another process has it. Rejects with `ModelRegistryBusyError` when the wait
|
|
139
|
+
* runs out, or with `options.cancelled()` when the signal aborts or the engine
|
|
140
|
+
* closes.
|
|
141
|
+
*/
|
|
142
|
+
export function acquireRegistryClient(options) {
|
|
143
|
+
const cancelled = options.cancelled ?? (() => new DownloadCancelledError());
|
|
144
|
+
const { signal } = options;
|
|
145
|
+
if (signal?.aborted || shuttingDown)
|
|
146
|
+
return Promise.reject(cancelled());
|
|
147
|
+
if (store && !yielding)
|
|
148
|
+
return Promise.resolve(takeLease(store));
|
|
149
|
+
return new Promise((resolve, reject) => {
|
|
150
|
+
const onAbort = () => {
|
|
151
|
+
if (!pending.delete(waiter))
|
|
152
|
+
return;
|
|
153
|
+
waiter.detach();
|
|
154
|
+
reject(cancelled());
|
|
155
|
+
if (pending.size === 0)
|
|
156
|
+
wakeOpener?.();
|
|
157
|
+
};
|
|
158
|
+
const waiter = {
|
|
159
|
+
wait: options.wait,
|
|
160
|
+
startedAt: Date.now(),
|
|
161
|
+
cancelled,
|
|
162
|
+
resolve,
|
|
163
|
+
reject,
|
|
164
|
+
detach: () => signal?.removeEventListener('abort', onAbort)
|
|
165
|
+
};
|
|
166
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
167
|
+
pending.add(waiter);
|
|
168
|
+
ensureOpener();
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
function takeLease(open) {
|
|
172
|
+
refs++;
|
|
173
|
+
const leaseGeneration = generation;
|
|
174
|
+
let released = false;
|
|
175
|
+
return {
|
|
176
|
+
client: open.client,
|
|
177
|
+
[Symbol.asyncDispose]: async () => {
|
|
178
|
+
if (released)
|
|
179
|
+
return;
|
|
180
|
+
released = true;
|
|
181
|
+
if (leaseGeneration !== generation)
|
|
182
|
+
return;
|
|
183
|
+
refs--;
|
|
184
|
+
if (refs === 0) {
|
|
185
|
+
idleSince = Date.now();
|
|
186
|
+
check();
|
|
57
187
|
}
|
|
58
|
-
|
|
59
|
-
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
function admitPending() {
|
|
192
|
+
if (!store)
|
|
193
|
+
return;
|
|
194
|
+
for (const waiter of [...pending]) {
|
|
195
|
+
pending.delete(waiter);
|
|
196
|
+
waiter.detach();
|
|
197
|
+
waiter.resolve(takeLease(store));
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function rejectPending(error) {
|
|
201
|
+
for (const waiter of [...pending]) {
|
|
202
|
+
pending.delete(waiter);
|
|
203
|
+
waiter.detach();
|
|
204
|
+
waiter.reject(error(waiter));
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
// Runs while the store is open: on every lease release and every poll.
|
|
208
|
+
function check() {
|
|
209
|
+
if (!store || !lifecycleActive())
|
|
210
|
+
return;
|
|
211
|
+
const othersWaiting = isMarkerHeld('waiting') === true;
|
|
212
|
+
if (refs > 0) {
|
|
213
|
+
if (othersWaiting && !yielding) {
|
|
214
|
+
yielding = true;
|
|
215
|
+
logger.info('Another QVAC process is waiting for the model registry; handing it over');
|
|
216
|
+
}
|
|
217
|
+
else if (!othersWaiting && yielding) {
|
|
218
|
+
// The other process stopped waiting: leases queued for the handover go ahead.
|
|
219
|
+
yielding = false;
|
|
220
|
+
admitPending();
|
|
221
|
+
}
|
|
222
|
+
// Leases queued behind the handover keep their deadlines.
|
|
223
|
+
expireWaiters('current', null);
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
if (othersWaiting) {
|
|
227
|
+
void closeStore('yield');
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
if (yielding) {
|
|
231
|
+
// The other process stopped waiting before this one finished.
|
|
232
|
+
yielding = false;
|
|
233
|
+
admitPending();
|
|
234
|
+
if (refs > 0)
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
if (Date.now() - idleSince >= timings.graceMs)
|
|
238
|
+
void closeStore('idle');
|
|
239
|
+
}
|
|
240
|
+
function closeStore(reason) {
|
|
241
|
+
const open = store;
|
|
242
|
+
store = null;
|
|
243
|
+
refs = 0;
|
|
244
|
+
generation++;
|
|
245
|
+
yielding = false;
|
|
246
|
+
if (watchTimer)
|
|
247
|
+
clearInterval(watchTimer);
|
|
248
|
+
watchTimer = null;
|
|
249
|
+
if (reason === 'yield')
|
|
250
|
+
handoffUntil = Date.now() + 2 * timings.pollMs;
|
|
251
|
+
logger.info(`🔌 Closing registry client (${reason})...`);
|
|
252
|
+
closing = (async () => {
|
|
253
|
+
try {
|
|
254
|
+
await open.client.close();
|
|
255
|
+
logger.info('✅ Registry client closed');
|
|
60
256
|
}
|
|
61
257
|
catch (error) {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
258
|
+
logger.error('❌ Error closing registry client:', error instanceof Error ? error.message : String(error));
|
|
259
|
+
}
|
|
260
|
+
finally {
|
|
261
|
+
unregisterCorestore(open.lifecycle);
|
|
262
|
+
open.leaseMarker?.release();
|
|
263
|
+
}
|
|
264
|
+
})().finally(() => {
|
|
265
|
+
closing = null;
|
|
266
|
+
if (pending.size > 0)
|
|
267
|
+
ensureOpener();
|
|
268
|
+
});
|
|
269
|
+
return closing;
|
|
270
|
+
}
|
|
271
|
+
// The runtime suspends and resumes the store through these. An idle store is
|
|
272
|
+
// closed rather than suspended, so another process can use it while this one is
|
|
273
|
+
// in the background. A store with leases in flight is suspended with them, and
|
|
274
|
+
// is handed over or closed once the runtime resumes.
|
|
275
|
+
async function suspendStore(open) {
|
|
276
|
+
if (store !== open)
|
|
277
|
+
return;
|
|
278
|
+
if (refs === 0)
|
|
279
|
+
await closeStore('suspend');
|
|
280
|
+
else
|
|
281
|
+
await open.client.suspend();
|
|
282
|
+
}
|
|
283
|
+
async function resumeStore(open) {
|
|
284
|
+
if (store === open)
|
|
285
|
+
await open.client.resume();
|
|
286
|
+
}
|
|
287
|
+
function ensureOpener() {
|
|
288
|
+
if (opener || closing || shuttingDown)
|
|
289
|
+
return;
|
|
290
|
+
if (store) {
|
|
291
|
+
if (!yielding)
|
|
292
|
+
admitPending();
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
opener = openWhenFree()
|
|
296
|
+
.catch((error) => rejectPending(() => error))
|
|
297
|
+
.finally(() => {
|
|
298
|
+
opener = null;
|
|
299
|
+
if (pending.size > 0 && !shuttingDown)
|
|
300
|
+
ensureOpener();
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
// Held shared while this process waits, so the holder knows to hand the store over.
|
|
304
|
+
class WaitingSignal {
|
|
305
|
+
marker = null;
|
|
306
|
+
raise() {
|
|
307
|
+
if (this.marker)
|
|
308
|
+
return;
|
|
309
|
+
// Fails for an instant while the holder checks; the next poll retries.
|
|
310
|
+
const marker = tryMarker('waiting', true);
|
|
311
|
+
if (typeof marker !== 'string')
|
|
312
|
+
this.marker = marker;
|
|
313
|
+
}
|
|
314
|
+
lower() {
|
|
315
|
+
this.marker?.release();
|
|
316
|
+
this.marker = null;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
async function openWhenFree() {
|
|
320
|
+
const waiting = new WaitingSignal();
|
|
321
|
+
let legacySince = null;
|
|
322
|
+
let announced = false;
|
|
323
|
+
try {
|
|
324
|
+
if (Date.now() < handoffUntil) {
|
|
325
|
+
waiting.raise();
|
|
326
|
+
await sleep(handoffUntil - Date.now());
|
|
327
|
+
}
|
|
328
|
+
while (pending.size > 0 && !shuttingDown) {
|
|
329
|
+
// A store opened while suspended would sync in the background, outside
|
|
330
|
+
// the lifecycle's suspend. Wait for the runtime to resume.
|
|
331
|
+
if (!lifecycleActive()) {
|
|
332
|
+
waiting.lower();
|
|
333
|
+
await sleep(timings.pollMs);
|
|
72
334
|
continue;
|
|
73
335
|
}
|
|
74
|
-
|
|
336
|
+
if (probeStore() === 'free') {
|
|
337
|
+
const opened = await tryOpen();
|
|
338
|
+
if (opened) {
|
|
339
|
+
waiting.lower();
|
|
340
|
+
if (shuttingDown) {
|
|
341
|
+
store = opened;
|
|
342
|
+
await closeStore('shutdown');
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
store = opened;
|
|
346
|
+
idleSince = Date.now();
|
|
347
|
+
watchTimer = setInterval(check, timings.pollMs);
|
|
348
|
+
if (announced)
|
|
349
|
+
logger.info('The model registry is free again; continuing');
|
|
350
|
+
admitPending();
|
|
351
|
+
if (refs === 0)
|
|
352
|
+
check();
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
waiting.raise();
|
|
357
|
+
const holder = isMarkerHeld('lease') === false ? 'legacy' : 'current';
|
|
358
|
+
if (holder === 'legacy')
|
|
359
|
+
legacySince ??= Date.now();
|
|
360
|
+
else
|
|
361
|
+
legacySince = null;
|
|
362
|
+
if (!announced) {
|
|
363
|
+
announced = true;
|
|
364
|
+
logger.info(holder === 'legacy'
|
|
365
|
+
? '⏳ An older QVAC process holds the model registry until it exits; waiting'
|
|
366
|
+
: '⏳ Another QVAC process is using the model registry; waiting for it to finish');
|
|
367
|
+
}
|
|
368
|
+
expireWaiters(holder, legacySince);
|
|
369
|
+
if (pending.size === 0)
|
|
370
|
+
return;
|
|
371
|
+
await sleep(timings.pollMs);
|
|
75
372
|
}
|
|
76
373
|
}
|
|
77
|
-
|
|
374
|
+
finally {
|
|
375
|
+
waiting.lower();
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
function budgetFor(wait) {
|
|
379
|
+
if (wait === 'query')
|
|
380
|
+
return timings.queryWaitMs;
|
|
381
|
+
if (wait === 'status')
|
|
382
|
+
return timings.statusWaitMs;
|
|
383
|
+
return Infinity;
|
|
78
384
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
385
|
+
function expireWaiters(holder, legacySince) {
|
|
386
|
+
const now = Date.now();
|
|
387
|
+
for (const waiter of [...pending]) {
|
|
388
|
+
const legacyFor = legacySince === null ? 0 : now - Math.max(legacySince, waiter.startedAt);
|
|
389
|
+
const waited = now - waiter.startedAt;
|
|
390
|
+
const expired = (holder === 'legacy' && legacyFor >= timings.legacyWaitMs) || waited >= budgetFor(waiter.wait);
|
|
391
|
+
if (!expired)
|
|
392
|
+
continue;
|
|
393
|
+
pending.delete(waiter);
|
|
394
|
+
waiter.detach();
|
|
395
|
+
waiter.reject(new ModelRegistryBusyError(holder, waited));
|
|
83
396
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
397
|
+
}
|
|
398
|
+
// `null` when another process has the store or took it first. The lease marker
|
|
399
|
+
// is taken before the store, so the store is never held by this version without
|
|
400
|
+
// it, even while a cold open syncs from the network.
|
|
401
|
+
async function tryOpen() {
|
|
402
|
+
const marker = tryMarker('lease', false);
|
|
403
|
+
// Another process on this version has the store, is opening it, or is
|
|
404
|
+
// checking the marker; the next poll looks again.
|
|
405
|
+
if (marker === 'held')
|
|
406
|
+
return null;
|
|
407
|
+
if (marker === 'unavailable') {
|
|
408
|
+
logger.warn('Cannot mark the model registry as held by this version; other processes will treat it as an older SDK');
|
|
87
409
|
}
|
|
410
|
+
const leaseMarker = marker === 'unavailable' ? null : marker;
|
|
88
411
|
logger.info('🔗 Creating new registry client...');
|
|
89
|
-
|
|
90
|
-
|
|
412
|
+
const client = new QVACRegistryClient({
|
|
413
|
+
registryCoreKey: DEFAULT_REGISTRY_CORE_KEY,
|
|
414
|
+
storage: storeDir(),
|
|
415
|
+
corestoreOpts: { wait: false }
|
|
91
416
|
});
|
|
92
|
-
return inflightInit;
|
|
93
|
-
}
|
|
94
|
-
export async function closeRegistryClient() {
|
|
95
|
-
if (!registryClient)
|
|
96
|
-
return;
|
|
97
|
-
const client = registryClient;
|
|
98
|
-
registryClient = null;
|
|
99
|
-
const corestore = client.corestore;
|
|
100
|
-
const hyperswarm = client.hyperswarm;
|
|
101
|
-
logger.info('🔌 Closing registry client...');
|
|
102
417
|
try {
|
|
103
|
-
await client.
|
|
104
|
-
logger.info('✅ Registry client closed');
|
|
418
|
+
await client.ready();
|
|
105
419
|
}
|
|
106
420
|
catch (error) {
|
|
107
|
-
|
|
421
|
+
// Released first: a half-open store still holds its lock, and the probe
|
|
422
|
+
// below would take this process for another.
|
|
423
|
+
await discardFailedOpen(client);
|
|
424
|
+
leaseMarker?.release();
|
|
425
|
+
if (isFdLockError(error) || probeStore() === 'busy') {
|
|
426
|
+
logger.debug('Registry store was taken by another process before this one opened it');
|
|
427
|
+
return null;
|
|
428
|
+
}
|
|
429
|
+
throw error;
|
|
430
|
+
}
|
|
431
|
+
const lifecycle = {
|
|
432
|
+
suspend: () => suspendStore(open),
|
|
433
|
+
resume: () => resumeStore(open)
|
|
434
|
+
};
|
|
435
|
+
const open = { client, leaseMarker, lifecycle };
|
|
436
|
+
registerCorestore(lifecycle, { label: 'registry-client', createdAt: Date.now() });
|
|
437
|
+
logger.info('✅ Registry client ready');
|
|
438
|
+
return open;
|
|
439
|
+
}
|
|
440
|
+
// A failed open never reaches the client's own close, so release what it had
|
|
441
|
+
// opened: the store with its file lock, and the swarm.
|
|
442
|
+
async function discardFailedOpen(client) {
|
|
443
|
+
const opened = client;
|
|
444
|
+
try {
|
|
445
|
+
await opened.hyperswarm?.destroy();
|
|
446
|
+
}
|
|
447
|
+
catch { }
|
|
448
|
+
try {
|
|
449
|
+
await opened.corestore?.close();
|
|
450
|
+
}
|
|
451
|
+
catch { }
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Close the store and reject every waiting lease. Leases still held are
|
|
455
|
+
* released by the close. Used by engine shutdown.
|
|
456
|
+
*/
|
|
457
|
+
export async function closeRegistryClient() {
|
|
458
|
+
shuttingDown = true;
|
|
459
|
+
try {
|
|
460
|
+
rejectPending((waiter) => waiter.cancelled());
|
|
461
|
+
wakeOpener?.();
|
|
462
|
+
if (opener)
|
|
463
|
+
await opener;
|
|
464
|
+
if (store)
|
|
465
|
+
await closeStore('shutdown');
|
|
466
|
+
if (closing)
|
|
467
|
+
await closing;
|
|
108
468
|
}
|
|
109
469
|
finally {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
if (hyperswarm)
|
|
113
|
-
unregisterSwarm(hyperswarm);
|
|
470
|
+
shuttingDown = false;
|
|
471
|
+
handoffUntil = 0;
|
|
114
472
|
}
|
|
115
473
|
}
|
|
116
474
|
export function hasActiveRegistryClient() {
|
|
117
|
-
return
|
|
475
|
+
return store !== null;
|
|
118
476
|
}
|
|
@@ -11,7 +11,7 @@ import type { Logger } from '../logging/types.js';
|
|
|
11
11
|
* The set is intentionally open-coded — adding a new kind is a one-line
|
|
12
12
|
* change and the union surfaces in editor autocomplete at every call site.
|
|
13
13
|
*/
|
|
14
|
-
export type RequestKind = 'completion' | 'batchCompletion' | 'embeddings' | 'decision' | 'rerank' | 'transcribe' | 'translate' | 'diffusion' | 'world' | 'audiogen' | 'tts' | 'ocr' | 'vla' | 'finetune' | 'loadModel' | 'downloadAsset' | 'rag' | 'rpcServer' | 'rpcDiscovery';
|
|
14
|
+
export type RequestKind = 'completion' | 'batchCompletion' | 'embeddings' | 'imageEmbedding' | 'decision' | 'rerank' | 'transcribe' | 'translate' | 'diffusion' | 'world' | 'audiogen' | 'tts' | 'ocr' | 'classification' | 'vla' | 'plugin' | 'vectorIndex' | 'finetune' | 'loadModel' | 'downloadAsset' | 'modelRegistry' | 'rag' | 'rpcServer' | 'rpcDiscovery';
|
|
15
15
|
/**
|
|
16
16
|
* Lifecycle states a request transitions through. A new context starts in
|
|
17
17
|
* `"running"`. `cancel(...)` flips it to `"cancelling"` and aborts the
|
|
@@ -42,6 +42,14 @@ export type RequestState = 'running' | 'cancelling' | 'completed' | 'failed' | '
|
|
|
42
42
|
*/
|
|
43
43
|
export interface RequestContext {
|
|
44
44
|
readonly requestId: string;
|
|
45
|
+
/** Caller session the request belongs to; the owner session when the request named none. */
|
|
46
|
+
readonly sessionId: string;
|
|
47
|
+
/**
|
|
48
|
+
* `(sessionId, requestId)` joined into the registry's storage key. Hand this,
|
|
49
|
+
* not `requestId`, to native code that keys jobs by id: two sessions may reuse
|
|
50
|
+
* one request id, and the addon has no session of its own to tell them apart.
|
|
51
|
+
*/
|
|
52
|
+
readonly registryKey: string;
|
|
45
53
|
readonly kind: RequestKind;
|
|
46
54
|
readonly modelId: string | undefined;
|
|
47
55
|
readonly signal: AbortSignal;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createRequestRegistry as createRegistry } from '../runtime/request-registry.js';
|
|
2
|
+
import { assertSessionOpen } from '../runtime/session-registry.js';
|
|
2
3
|
/**
|
|
3
4
|
* Shared singleton. Every long-running request registers under this
|
|
4
5
|
* registry, so a `cancel({ requestId })` call can find its target without
|
|
@@ -21,6 +22,12 @@ let registry = null;
|
|
|
21
22
|
// passes an infinite cap and never enters the lane, so it stays ungated.
|
|
22
23
|
const LLAMACPP_COMPLETION_SLOT_GROUP = 'llamacppCompletion';
|
|
23
24
|
function installDefaultPolicies(r) {
|
|
25
|
+
r.policy({
|
|
26
|
+
kind: 'imageEmbedding',
|
|
27
|
+
maxConcurrentPerModel: 1,
|
|
28
|
+
onOverflow: 'queue',
|
|
29
|
+
maxQueueDepthPerModel: 64
|
|
30
|
+
});
|
|
24
31
|
// Laya owns one active job per model. Wait in the registry so cancelling a
|
|
25
32
|
// queued request never reaches the addon or interrupts the active job.
|
|
26
33
|
r.policy({
|
|
@@ -36,6 +43,13 @@ function installDefaultPolicies(r) {
|
|
|
36
43
|
onOverflow: 'queue',
|
|
37
44
|
maxQueueDepthPerModel: 64
|
|
38
45
|
});
|
|
46
|
+
// Embeddings have one native job and model-wide cancellation. Preserve
|
|
47
|
+
// busy rejection, before media preparation or native cancellation is wired.
|
|
48
|
+
r.policy({
|
|
49
|
+
kind: 'embeddings',
|
|
50
|
+
maxConcurrentPerModel: 1,
|
|
51
|
+
onOverflow: 'reject'
|
|
52
|
+
});
|
|
39
53
|
// Cap is the model's `parallel`, passed per request by the handlers; the value
|
|
40
54
|
// here is only the fallback when a caller supplies none.
|
|
41
55
|
r.policy({
|
|
@@ -97,6 +111,16 @@ function installDefaultPolicies(r) {
|
|
|
97
111
|
onOverflow: 'queue',
|
|
98
112
|
maxQueueDepthPerModel: 256
|
|
99
113
|
});
|
|
114
|
+
// Whisper, Parakeet and MOSS run one job per model and cancel it model-wide,
|
|
115
|
+
// with no job id. A second request waiting inside the addon would stop the
|
|
116
|
+
// active one when it is cancelled, which crosses sessions. Waiting in the
|
|
117
|
+
// registry instead keeps a cancelled waiter away from the addon.
|
|
118
|
+
r.policy({
|
|
119
|
+
kind: 'transcribe',
|
|
120
|
+
maxConcurrentPerModel: 1,
|
|
121
|
+
onOverflow: 'queue',
|
|
122
|
+
maxQueueDepthPerModel: 64
|
|
123
|
+
});
|
|
100
124
|
// Diffusion has one native job per model and model-wide cancellation.
|
|
101
125
|
r.policy({
|
|
102
126
|
kind: 'diffusion',
|
|
@@ -118,7 +142,7 @@ function installDefaultPolicies(r) {
|
|
|
118
142
|
}
|
|
119
143
|
export function getRequestRegistry() {
|
|
120
144
|
if (!registry) {
|
|
121
|
-
registry = createRegistry();
|
|
145
|
+
registry = createRegistry({ assertSessionOpen });
|
|
122
146
|
installDefaultPolicies(registry);
|
|
123
147
|
}
|
|
124
148
|
return registry;
|