@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
package/dist/runtime/env.js
CHANGED
|
@@ -4,30 +4,15 @@ const envSchema = z.object({
|
|
|
4
4
|
HOME_DIR: z.string()
|
|
5
5
|
});
|
|
6
6
|
let validatedEnv = null;
|
|
7
|
+
function defaultHomeDir() {
|
|
8
|
+
// Snap's HOME can be revision-scoped; SNAP_USER_COMMON is stable.
|
|
9
|
+
return env['SNAP_USER_COMMON'] ?? env['HOME'] ?? env['USERPROFILE'] ?? '/tmp';
|
|
10
|
+
}
|
|
7
11
|
/**
|
|
8
12
|
* Initialize the environment. Call once at startup.
|
|
9
13
|
*/
|
|
10
|
-
export function initEnv() {
|
|
11
|
-
|
|
12
|
-
// Snap's HOME can be revision-scoped; SNAP_USER_COMMON is stable.
|
|
13
|
-
env['SNAP_USER_COMMON'] ?? env['HOME'] ?? env['USERPROFILE'] ?? '/tmp';
|
|
14
|
-
let envConfig = {
|
|
15
|
-
HOME_DIR: defaultHomeDir
|
|
16
|
-
};
|
|
17
|
-
const isBareKit = typeof globalThis.BareKit !== 'undefined';
|
|
18
|
-
if (isBareKit && Bare.argv[0]) {
|
|
19
|
-
envConfig['HOME_DIR'] = Bare.argv[0];
|
|
20
|
-
}
|
|
21
|
-
if (Bare.argv[2]) {
|
|
22
|
-
try {
|
|
23
|
-
const overrides = JSON.parse(Bare.argv[2]);
|
|
24
|
-
envConfig = { ...envConfig, ...overrides };
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
// Ignore non-JSON args
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
validatedEnv = envSchema.parse(envConfig);
|
|
14
|
+
export function initEnv(options = {}) {
|
|
15
|
+
validatedEnv = envSchema.parse({ HOME_DIR: options.homeDir ?? defaultHomeDir() });
|
|
31
16
|
}
|
|
32
17
|
/**
|
|
33
18
|
* Get the engine environment. Must call initEnv() first.
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -2,4 +2,7 @@ export { createDisposableScope } from '../runtime/disposable-scope.js';
|
|
|
2
2
|
export type { DisposableScope } from '../runtime/disposable-scope.js';
|
|
3
3
|
export type { RequestContext, RequestKind, RequestState, RequestLogContext } from '../runtime/request-context.js';
|
|
4
4
|
export { createRequestRegistry, getRequestRegistry, withRequestContext } from '../runtime/request-context.js';
|
|
5
|
-
export
|
|
5
|
+
export { OWNER_SESSION_ID, resolveSessionId, requestRegistryKey } from '../runtime/session-id.js';
|
|
6
|
+
export { openSession, closeSession, assertSessionOpen, assertWireSessionOpen, isSessionOpen, isOwnerSession, listSessions, hasOpenSessions, clearSessions } from '../runtime/session-registry.js';
|
|
7
|
+
export type { SessionEntry } from '../runtime/session-registry.js';
|
|
8
|
+
export type { BeginOpts, CancelByModelId, CancelByRequestId, CancelBySession, CancelTarget, ConcurrencyPolicy, ManagedRequestContext, RequestOutcome, RequestRegistry } from '../runtime/request-registry.js';
|
package/dist/runtime/index.js
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
export { createDisposableScope } from '../runtime/disposable-scope.js';
|
|
2
2
|
export { createRequestRegistry, getRequestRegistry, withRequestContext } from '../runtime/request-context.js';
|
|
3
|
+
export { OWNER_SESSION_ID, resolveSessionId, requestRegistryKey } from '../runtime/session-id.js';
|
|
4
|
+
export { openSession, closeSession, assertSessionOpen, assertWireSessionOpen, isSessionOpen, isOwnerSession, listSessions, hasOpenSessions, clearSessions } from '../runtime/session-registry.js';
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { type EnvOptions } from '../runtime/env.js';
|
|
2
|
+
export type InitializeOptions = EnvOptions;
|
|
1
3
|
/**
|
|
2
|
-
* Initialize the engine: start log buffering,
|
|
3
|
-
* lock (guards the cache dir against a second instance in the same
|
|
4
|
-
* Idempotent.
|
|
4
|
+
* Initialize the engine: start log buffering, set the home folder, and acquire
|
|
5
|
+
* the cache lock (guards the cache dir against a second instance in the same
|
|
6
|
+
* home). Idempotent: options are only read by the first call.
|
|
5
7
|
*/
|
|
6
|
-
export declare function initialize(): void;
|
|
8
|
+
export declare function initialize(options?: InitializeOptions): void;
|
|
7
9
|
export declare function isInitialized(): boolean;
|
|
8
10
|
/**
|
|
9
11
|
* Pre-terminate cleanup, callable while the isolate is still alive. Runs the
|
|
@@ -8,6 +8,7 @@ import { closeRegistryClient } from '../runtime/registry-client.js';
|
|
|
8
8
|
import { clearAllLoggingStreams, startLogBuffering, stopLogBufferingWithTimeout } from '../runtime/logging-stream-registry.js';
|
|
9
9
|
import { clearAllAddonLoggers, getEngineLogger, LOG_ID, ALL_LOG_ID } from '../logging/index.js';
|
|
10
10
|
import { clearPlugins } from '../plugins/index.js';
|
|
11
|
+
import { clearSessions } from '../runtime/session-registry.js';
|
|
11
12
|
import { acquireCacheLock, releaseCacheLock } from '../runtime/cache-lock.js';
|
|
12
13
|
import { nativeResourceCollectorDependencies } from '../resources/native.js';
|
|
13
14
|
import { destroyResourceCollector, initializeResourceCollector } from '../resources/instance.js';
|
|
@@ -19,16 +20,16 @@ let cleanupRan = false;
|
|
|
19
20
|
let cleanupPromise = null;
|
|
20
21
|
const logger = getEngineLogger();
|
|
21
22
|
/**
|
|
22
|
-
* Initialize the engine: start log buffering,
|
|
23
|
-
* lock (guards the cache dir against a second instance in the same
|
|
24
|
-
* Idempotent.
|
|
23
|
+
* Initialize the engine: start log buffering, set the home folder, and acquire
|
|
24
|
+
* the cache lock (guards the cache dir against a second instance in the same
|
|
25
|
+
* home). Idempotent: options are only read by the first call.
|
|
25
26
|
*/
|
|
26
|
-
export function initialize() {
|
|
27
|
+
export function initialize(options = {}) {
|
|
27
28
|
if (initialized)
|
|
28
29
|
return;
|
|
29
30
|
startLogBuffering(LOG_ID);
|
|
30
31
|
startLogBuffering(ALL_LOG_ID);
|
|
31
|
-
initEnv();
|
|
32
|
+
initEnv(options);
|
|
32
33
|
acquireCacheLock();
|
|
33
34
|
initializeResourceCollector(nativeResourceCollectorDependencies);
|
|
34
35
|
// A `subscribeServerLogs` consumer can attach right away. Bound the global
|
|
@@ -61,6 +62,9 @@ async function runCleanup() {
|
|
|
61
62
|
return;
|
|
62
63
|
cleanupPromise = (async () => {
|
|
63
64
|
destroyResourceCollector();
|
|
65
|
+
// Root shutdown ends every caller session; their models go with
|
|
66
|
+
// `unloadAllModels` below and their requests with the engine.
|
|
67
|
+
clearSessions();
|
|
64
68
|
clearRegistries();
|
|
65
69
|
await Promise.allSettled([
|
|
66
70
|
disposeAllVectorIndexes(),
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { imageEmbeddingConfigSchema } from '../schemas/image-embedding.js';
|
|
1
2
|
import { ModelType } from '../schemas/index.js';
|
|
2
3
|
import { llmConfigSchema, embedConfigSchema } from '../schemas/llamacpp-config.js';
|
|
3
4
|
import { whisperConfigSchema, parakeetRuntimeConfigSchema, mossTranscribeConfigSchema } from '../schemas/transcription-config.js';
|
|
@@ -14,6 +15,7 @@ export const CANONICAL_TO_ALIAS = {
|
|
|
14
15
|
[ModelType.llamacppEmbedding]: 'embeddings',
|
|
15
16
|
[ModelType.llamacppDecision]: 'llamacpp-decision',
|
|
16
17
|
[ModelType.llamacppRerank]: 'llamacpp-rerank',
|
|
18
|
+
[ModelType.ggmlImageEmbedding]: 'ggml-image-embedding',
|
|
17
19
|
[ModelType.whispercppTranscription]: 'whisper',
|
|
18
20
|
[ModelType.bciWhispercppTranscription]: 'bci',
|
|
19
21
|
[ModelType.parakeetTranscription]: 'parakeet',
|
|
@@ -32,6 +34,7 @@ export const MODEL_CONFIG_SCHEMAS = {
|
|
|
32
34
|
[ModelType.llamacppEmbedding]: embedConfigSchema,
|
|
33
35
|
[ModelType.llamacppDecision]: decisionConfigSchema,
|
|
34
36
|
[ModelType.llamacppRerank]: rerankConfigSchema,
|
|
37
|
+
[ModelType.ggmlImageEmbedding]: imageEmbeddingConfigSchema,
|
|
35
38
|
[ModelType.whispercppTranscription]: whisperConfigSchema,
|
|
36
39
|
[ModelType.bciWhispercppTranscription]: bciConfigSchema,
|
|
37
40
|
[ModelType.parakeetTranscription]: parakeetRuntimeConfigSchema.passthrough(),
|
|
@@ -62,7 +65,8 @@ export const BUILTIN_DEVICE_PATTERNS = [
|
|
|
62
65
|
},
|
|
63
66
|
defaults: {
|
|
64
67
|
[ModelType.llamacppCompletion]: { device: 'cpu' },
|
|
65
|
-
[ModelType.llamacppEmbedding]: { device: 'cpu' }
|
|
68
|
+
[ModelType.llamacppEmbedding]: { device: 'cpu' },
|
|
69
|
+
[ModelType.llamacppRerank]: { device: 'cpu' }
|
|
66
70
|
}
|
|
67
71
|
}
|
|
68
72
|
];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CanonicalModelType,
|
|
1
|
+
import type { CanonicalModelType, ModelFitProbe, ToolDialect } from '../schemas/index.js';
|
|
2
2
|
interface AddonInterface {
|
|
3
3
|
cancel(jobId?: string): Promise<void>;
|
|
4
4
|
}
|
|
@@ -24,14 +24,22 @@ interface LocalOptions {
|
|
|
24
24
|
path: string;
|
|
25
25
|
loadedAt: Date;
|
|
26
26
|
config: unknown;
|
|
27
|
+
/** The configuration the model was loaded with, which its id is a hash of. */
|
|
28
|
+
loadedConfig: unknown;
|
|
27
29
|
modelType: CanonicalModelType;
|
|
28
30
|
name?: string | undefined;
|
|
29
|
-
fitProbe?:
|
|
31
|
+
fitProbe?: ModelFitProbe | undefined;
|
|
30
32
|
toolDialect?: ToolDialect | undefined;
|
|
31
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* `leases` holds the ids of the caller sessions that loaded the model and have
|
|
36
|
+
* not unloaded it. The model leaves memory when the set empties or on root
|
|
37
|
+
* shutdown.
|
|
38
|
+
*/
|
|
32
39
|
export type ModelEntry = {
|
|
33
40
|
id: string;
|
|
34
41
|
local: LocalOptions;
|
|
42
|
+
leases: Set<string>;
|
|
35
43
|
};
|
|
36
44
|
export declare function registerModel(id: string, options: {
|
|
37
45
|
model: AnyModel;
|
|
@@ -39,10 +47,20 @@ export declare function registerModel(id: string, options: {
|
|
|
39
47
|
config: unknown;
|
|
40
48
|
modelType: CanonicalModelType;
|
|
41
49
|
name?: string | undefined;
|
|
42
|
-
fitProbe?:
|
|
50
|
+
fitProbe?: ModelFitProbe | undefined;
|
|
43
51
|
toolDialect?: ToolDialect | undefined;
|
|
44
|
-
}): void;
|
|
52
|
+
}, sessionId?: string): void;
|
|
45
53
|
export declare function getModelEntry(id: string): ModelEntry | null;
|
|
54
|
+
/** Records that `sessionId` holds the model. Idempotent per session. */
|
|
55
|
+
export declare function addModelLease(id: string, sessionId: string): void;
|
|
56
|
+
/**
|
|
57
|
+
* Drops `sessionId`'s lease. Returns the number of leases left, or `null` when
|
|
58
|
+
* the session held none, so the caller can tell "released, model stays" from
|
|
59
|
+
* "nothing to release".
|
|
60
|
+
*/
|
|
61
|
+
export declare function releaseModelLease(id: string, sessionId: string): number | null;
|
|
62
|
+
/** Ids of the models `sessionId` holds a lease on. */
|
|
63
|
+
export declare function getModelIdsLeasedBy(sessionId: string): string[];
|
|
46
64
|
export declare function getModel(id: string): AnyModel;
|
|
47
65
|
export declare function isModelLoaded(id: string): boolean;
|
|
48
66
|
export declare function unregisterModel(id: string): ModelEntry | null;
|
|
@@ -55,6 +73,12 @@ export declare function getModelInfo(id: string): {
|
|
|
55
73
|
name?: string;
|
|
56
74
|
} | null;
|
|
57
75
|
export declare function getModelConfig(id: string): unknown;
|
|
76
|
+
/**
|
|
77
|
+
* Whether a reload left the model with a configuration other than the one it
|
|
78
|
+
* was loaded with. Its id still hashes the loaded configuration, so a load of
|
|
79
|
+
* that configuration would otherwise get the reconfigured model.
|
|
80
|
+
*/
|
|
81
|
+
export declare function isModelReconfigured(id: string): boolean;
|
|
58
82
|
export declare function updateModelConfig(id: string, config: unknown): void;
|
|
59
83
|
export declare function clearRegistry(): void;
|
|
60
84
|
export declare function getRegistryStats(): {
|
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
import { ModelAlreadyRegisteredError, ModelNotFoundError } from '../errors/index.js';
|
|
2
2
|
import { getEngineLogger } from '../logging/index.js';
|
|
3
|
+
import { OWNER_SESSION_ID } from '../runtime/session-id.js';
|
|
4
|
+
import { canonicalConfigString } from '../utils/formatting.js';
|
|
3
5
|
const logger = getEngineLogger();
|
|
4
6
|
// Global registry state - using stateless functions to manage it
|
|
5
7
|
const modelRegistry = new Map();
|
|
6
|
-
export function registerModel(id, options) {
|
|
8
|
+
export function registerModel(id, options, sessionId = OWNER_SESSION_ID) {
|
|
7
9
|
if (modelRegistry.has(id)) {
|
|
8
10
|
throw new ModelAlreadyRegisteredError(id);
|
|
9
11
|
}
|
|
10
12
|
modelRegistry.set(id, {
|
|
11
13
|
id,
|
|
14
|
+
leases: new Set([sessionId]),
|
|
12
15
|
local: {
|
|
13
16
|
model: options.model,
|
|
14
17
|
path: options.path,
|
|
15
18
|
loadedAt: new Date(),
|
|
16
19
|
config: options.config,
|
|
20
|
+
loadedConfig: options.config,
|
|
17
21
|
modelType: options.modelType,
|
|
18
22
|
name: options.name,
|
|
19
23
|
fitProbe: options.fitProbe,
|
|
@@ -26,6 +30,35 @@ export function registerModel(id, options) {
|
|
|
26
30
|
export function getModelEntry(id) {
|
|
27
31
|
return modelRegistry.get(id) || null;
|
|
28
32
|
}
|
|
33
|
+
/** Records that `sessionId` holds the model. Idempotent per session. */
|
|
34
|
+
export function addModelLease(id, sessionId) {
|
|
35
|
+
const entry = modelRegistry.get(id);
|
|
36
|
+
if (!entry) {
|
|
37
|
+
throw new ModelNotFoundError(id);
|
|
38
|
+
}
|
|
39
|
+
entry.leases.add(sessionId);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Drops `sessionId`'s lease. Returns the number of leases left, or `null` when
|
|
43
|
+
* the session held none, so the caller can tell "released, model stays" from
|
|
44
|
+
* "nothing to release".
|
|
45
|
+
*/
|
|
46
|
+
export function releaseModelLease(id, sessionId) {
|
|
47
|
+
const entry = modelRegistry.get(id);
|
|
48
|
+
if (!entry || !entry.leases.has(sessionId))
|
|
49
|
+
return null;
|
|
50
|
+
entry.leases.delete(sessionId);
|
|
51
|
+
return entry.leases.size;
|
|
52
|
+
}
|
|
53
|
+
/** Ids of the models `sessionId` holds a lease on. */
|
|
54
|
+
export function getModelIdsLeasedBy(sessionId) {
|
|
55
|
+
const ids = [];
|
|
56
|
+
for (const entry of modelRegistry.values()) {
|
|
57
|
+
if (entry.leases.has(sessionId))
|
|
58
|
+
ids.push(entry.id);
|
|
59
|
+
}
|
|
60
|
+
return ids;
|
|
61
|
+
}
|
|
29
62
|
export function getModel(id) {
|
|
30
63
|
const entry = modelRegistry.get(id);
|
|
31
64
|
if (!entry) {
|
|
@@ -71,6 +104,18 @@ export function getModelConfig(id) {
|
|
|
71
104
|
}
|
|
72
105
|
return entry.local.config;
|
|
73
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* Whether a reload left the model with a configuration other than the one it
|
|
109
|
+
* was loaded with. Its id still hashes the loaded configuration, so a load of
|
|
110
|
+
* that configuration would otherwise get the reconfigured model.
|
|
111
|
+
*/
|
|
112
|
+
export function isModelReconfigured(id) {
|
|
113
|
+
const entry = modelRegistry.get(id);
|
|
114
|
+
if (!entry)
|
|
115
|
+
return false;
|
|
116
|
+
return (canonicalConfigString(entry.local.config) !==
|
|
117
|
+
canonicalConfigString(entry.local.loadedConfig));
|
|
118
|
+
}
|
|
74
119
|
export function updateModelConfig(id, config) {
|
|
75
120
|
const entry = modelRegistry.get(id);
|
|
76
121
|
if (!entry) {
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
/**
|
|
3
|
+
* Runs `task` as the only load, reload or unload of `modelId`. A caller that
|
|
4
|
+
* arrives while one is running waits for it to settle before running its own
|
|
5
|
+
* task, so it sees the model registered (and takes a lease) or finds it absent
|
|
6
|
+
* (and loads it itself after a failed load or a finished unload). A caller
|
|
7
|
+
* whose request is cancelled while it waits leaves the queue at once with
|
|
8
|
+
* `InferenceCancelledError`; the work it waited for is not touched.
|
|
9
|
+
*/
|
|
10
|
+
export declare function withModelWorkLock<T>(modelId: string, task: () => Promise<T>, request?: {
|
|
11
|
+
signal: AbortSignal;
|
|
12
|
+
requestId: string;
|
|
13
|
+
}): Promise<T>;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { InferenceCancelledError } from '../errors/index.js';
|
|
2
|
+
// A load, reload or last-lease unload in progress per model id. Two sessions
|
|
3
|
+
// loading the same configuration at once would both pass the loaded check and
|
|
4
|
+
// both load natively; the second native model could never be registered or
|
|
5
|
+
// unloaded. The id is a hash of the configuration, so one native load per id
|
|
6
|
+
// is the rule, and a lease is taken only once the model is registered or
|
|
7
|
+
// reconfigured.
|
|
8
|
+
const modelWorkInFlight = new Map();
|
|
9
|
+
/**
|
|
10
|
+
* Runs `task` as the only load, reload or unload of `modelId`. A caller that
|
|
11
|
+
* arrives while one is running waits for it to settle before running its own
|
|
12
|
+
* task, so it sees the model registered (and takes a lease) or finds it absent
|
|
13
|
+
* (and loads it itself after a failed load or a finished unload). A caller
|
|
14
|
+
* whose request is cancelled while it waits leaves the queue at once with
|
|
15
|
+
* `InferenceCancelledError`; the work it waited for is not touched.
|
|
16
|
+
*/
|
|
17
|
+
export async function withModelWorkLock(modelId, task, request) {
|
|
18
|
+
let pending = modelWorkInFlight.get(modelId);
|
|
19
|
+
while (pending !== undefined) {
|
|
20
|
+
await (request ? untilSettledOrAborted(pending, request.signal) : pending);
|
|
21
|
+
if (request?.signal.aborted === true)
|
|
22
|
+
throw new InferenceCancelledError(request.requestId);
|
|
23
|
+
pending = modelWorkInFlight.get(modelId);
|
|
24
|
+
}
|
|
25
|
+
const run = task();
|
|
26
|
+
modelWorkInFlight.set(modelId, run.then(() => undefined, () => undefined));
|
|
27
|
+
try {
|
|
28
|
+
return await run;
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
modelWorkInFlight.delete(modelId);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function untilSettledOrAborted(pending, signal) {
|
|
35
|
+
if (signal.aborted)
|
|
36
|
+
return Promise.resolve();
|
|
37
|
+
return new Promise((resolve) => {
|
|
38
|
+
const onAbort = () => resolve();
|
|
39
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
40
|
+
void pending.then(() => {
|
|
41
|
+
signal.removeEventListener('abort', onAbort);
|
|
42
|
+
resolve();
|
|
43
|
+
}, () => {
|
|
44
|
+
signal.removeEventListener('abort', onAbort);
|
|
45
|
+
resolve();
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
}
|
|
@@ -1,4 +1,50 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
1
2
|
import { QVACRegistryClient } from '@qvac/registry-client';
|
|
2
|
-
export
|
|
3
|
+
export interface RegistryClientTimings {
|
|
4
|
+
/** How long the store stays open after the last lease ends. */
|
|
5
|
+
graceMs: number;
|
|
6
|
+
/** Cross-process poll interval, and how often the holder checks for waiters. */
|
|
7
|
+
pollMs: number;
|
|
8
|
+
/** How long a query waits for the store before giving up. */
|
|
9
|
+
queryWaitMs: number;
|
|
10
|
+
/** How long a cache-status lookup waits before it gives up and reports what is on disk. */
|
|
11
|
+
statusWaitMs: number;
|
|
12
|
+
/** How long any operation waits for an older SDK that holds the store until it exits. */
|
|
13
|
+
legacyWaitMs: number;
|
|
14
|
+
}
|
|
15
|
+
/** Test seam: shorten the timings. Called without arguments, restores the defaults. */
|
|
16
|
+
export declare function setRegistryClientTimingsForTest(next?: Partial<RegistryClientTimings>): void;
|
|
17
|
+
/**
|
|
18
|
+
* `download` waits for a holder on this SDK version until the caller cancels.
|
|
19
|
+
* `query` gives up after `queryWaitMs`, and `status` after `statusWaitMs`. All
|
|
20
|
+
* give up after `legacyWaitMs` when the holder is an older SDK.
|
|
21
|
+
*/
|
|
22
|
+
export type RegistryWait = 'download' | 'query' | 'status';
|
|
23
|
+
export interface AcquireRegistryClientOptions {
|
|
24
|
+
wait: RegistryWait;
|
|
25
|
+
/** Aborting stops the wait. A lease already granted is unaffected. */
|
|
26
|
+
signal?: AbortSignal | undefined;
|
|
27
|
+
/** The error a cancelled wait rejects with. Defaults to `DownloadCancelledError`. */
|
|
28
|
+
cancelled?: () => Error;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The registry client, held open until disposed. Acquire one per operation and
|
|
32
|
+
* never acquire a second while holding one: while another process waits for the
|
|
33
|
+
* store, new leases wait for this process to hand it over.
|
|
34
|
+
*/
|
|
35
|
+
export interface RegistryClientLease extends AsyncDisposable {
|
|
36
|
+
readonly client: QVACRegistryClient;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Lease the registry client, opening the store if needed and waiting while
|
|
40
|
+
* another process has it. Rejects with `ModelRegistryBusyError` when the wait
|
|
41
|
+
* runs out, or with `options.cancelled()` when the signal aborts or the engine
|
|
42
|
+
* closes.
|
|
43
|
+
*/
|
|
44
|
+
export declare function acquireRegistryClient(options: AcquireRegistryClientOptions): Promise<RegistryClientLease>;
|
|
45
|
+
/**
|
|
46
|
+
* Close the store and reject every waiting lease. Leases still held are
|
|
47
|
+
* released by the close. Used by engine shutdown.
|
|
48
|
+
*/
|
|
3
49
|
export declare function closeRegistryClient(): Promise<void>;
|
|
4
50
|
export declare function hasActiveRegistryClient(): boolean;
|