@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
/** Whether this process holds the lock of any of these keys. */
|
|
3
|
+
export declare function holdsDownloadLockHere(resourceKeys: string[]): boolean;
|
|
4
|
+
/**
|
|
5
|
+
* Wait until this process holds none of these keys' locks, such as a cancelled
|
|
6
|
+
* download still cleaning up. False if one is still held after `timeoutMs`.
|
|
7
|
+
*/
|
|
8
|
+
export declare function waitForDownloadLocksHere(resourceKeys: string[], timeoutMs: number): Promise<boolean>;
|
|
9
|
+
/**
|
|
10
|
+
* Hold the cross-process lock for one cached resource until disposed, waiting
|
|
11
|
+
* while another holder has it. The OS releases it if the holder exits.
|
|
12
|
+
* Cancelling while waiting rejects with `DownloadCancelledError`.
|
|
13
|
+
*/
|
|
14
|
+
export declare function acquireDownloadLock(resourceKey: string, signal: AbortSignal): Promise<AsyncDisposable>;
|
|
15
|
+
/** Take every key's lock without waiting, or none; `null` while any is held. */
|
|
16
|
+
export declare function tryAcquireDownloadLocks(resourceKeys: string[]): Promise<AsyncDisposable | null>;
|
|
17
|
+
/** Where a download is written until it is complete and verified. */
|
|
18
|
+
export declare function partialDownloadPath(finalPath: string): string;
|
|
19
|
+
/** Atomically move a verified download into place. */
|
|
20
|
+
export declare function publishDownload(finalPath: string): Promise<void>;
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import fs, { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import fsx from 'fs-native-extensions';
|
|
4
|
+
import { generateShortHash, getModelsCacheDir, renameWithCachedChecksum } from '../../utils/index.js';
|
|
5
|
+
import { DownloadCancelledError } from '../../errors/index.js';
|
|
6
|
+
import { getEngineLogger } from '../../logging/index.js';
|
|
7
|
+
const logger = getEngineLogger();
|
|
8
|
+
const LOCK_DIR_NAME = '.locks';
|
|
9
|
+
const LOCK_POLL_MS = 250;
|
|
10
|
+
const NO_WRITE_ACCESS = new Set(['EACCES', 'EPERM', 'EROFS']);
|
|
11
|
+
// The OS grants a lock per open handle, so this process's own holders also make
|
|
12
|
+
// `tryAcquireDownloadLocks` fail. Tracking them tells them apart from other
|
|
13
|
+
// processes. Each entry settles once its lock is released.
|
|
14
|
+
const heldHere = new Map();
|
|
15
|
+
function trackHeldHere(resourceKey, lock) {
|
|
16
|
+
let markReleased;
|
|
17
|
+
const released = new Promise((resolve) => {
|
|
18
|
+
markReleased = resolve;
|
|
19
|
+
});
|
|
20
|
+
const holders = heldHere.get(resourceKey) ?? new Set();
|
|
21
|
+
holders.add(released);
|
|
22
|
+
heldHere.set(resourceKey, holders);
|
|
23
|
+
return {
|
|
24
|
+
fd: lock.fd,
|
|
25
|
+
[Symbol.asyncDispose]: async () => {
|
|
26
|
+
await lock[Symbol.asyncDispose]();
|
|
27
|
+
holders.delete(released);
|
|
28
|
+
if (holders.size === 0 && heldHere.get(resourceKey) === holders)
|
|
29
|
+
heldHere.delete(resourceKey);
|
|
30
|
+
markReleased();
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/** Whether this process holds the lock of any of these keys. */
|
|
35
|
+
export function holdsDownloadLockHere(resourceKeys) {
|
|
36
|
+
return resourceKeys.some((key) => heldHere.has(key));
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Wait until this process holds none of these keys' locks, such as a cancelled
|
|
40
|
+
* download still cleaning up. False if one is still held after `timeoutMs`.
|
|
41
|
+
*/
|
|
42
|
+
export async function waitForDownloadLocksHere(resourceKeys, timeoutMs) {
|
|
43
|
+
const deadline = Date.now() + timeoutMs;
|
|
44
|
+
for (;;) {
|
|
45
|
+
const pending = resourceKeys.flatMap((key) => [...(heldHere.get(key) ?? [])]);
|
|
46
|
+
if (pending.length === 0)
|
|
47
|
+
return true;
|
|
48
|
+
const remaining = deadline - Date.now();
|
|
49
|
+
if (remaining <= 0)
|
|
50
|
+
return false;
|
|
51
|
+
let timer;
|
|
52
|
+
const timedOut = await Promise.race([
|
|
53
|
+
Promise.all(pending).then(() => false),
|
|
54
|
+
new Promise((resolve) => {
|
|
55
|
+
timer = setTimeout(() => resolve(true), remaining);
|
|
56
|
+
})
|
|
57
|
+
]);
|
|
58
|
+
clearTimeout(timer);
|
|
59
|
+
if (timedOut)
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
// `null` when this process can change nothing in the models directory, such as
|
|
64
|
+
// a prefilled read-only cache: it can then only load cached models. Anywhere it
|
|
65
|
+
// could write, a lock that cannot be created fails instead of running unlocked.
|
|
66
|
+
async function openLockFile(resourceKey) {
|
|
67
|
+
const modelsDir = getModelsCacheDir();
|
|
68
|
+
const lockDir = path.join(modelsDir, LOCK_DIR_NAME);
|
|
69
|
+
let handle;
|
|
70
|
+
try {
|
|
71
|
+
await fsPromises.mkdir(lockDir, { recursive: true });
|
|
72
|
+
// Not 'a': Windows refuses to lock a handle opened for append only.
|
|
73
|
+
handle = await fsPromises.open(path.join(lockDir, `${generateShortHash(resourceKey)}.lock`), 'a+');
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
if (!NO_WRITE_ACCESS.has(error.code ?? ''))
|
|
77
|
+
throw error;
|
|
78
|
+
if (await canChangeModels(modelsDir)) {
|
|
79
|
+
logger.error(`❌ Cannot lock ${resourceKey}: its lock file in ${lockDir} is not writable`, {
|
|
80
|
+
error
|
|
81
|
+
});
|
|
82
|
+
throw error;
|
|
83
|
+
}
|
|
84
|
+
logger.warn(`⚠️ The models directory is read-only; reading ${resourceKey} without a lock`, {
|
|
85
|
+
error
|
|
86
|
+
});
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
fd: handle.fd,
|
|
91
|
+
// Closing the descriptor releases its lock.
|
|
92
|
+
[Symbol.asyncDispose]: () => handle.close().catch((error) => {
|
|
93
|
+
logger.debug('Failed to close download lock', { resourceKey, error });
|
|
94
|
+
})
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
// Downloads and deletes only add, rename or remove directory entries and write
|
|
98
|
+
// `.part` files, so a process can change a model only through a writable folder
|
|
99
|
+
// or `.part` file anywhere under `dir`. Lock files cannot change a model. A link
|
|
100
|
+
// counts if its target is writable; it is not entered, so cycles cannot loop.
|
|
101
|
+
async function canChangeModels(dir, isRoot = true) {
|
|
102
|
+
if (await hasAccess(dir, fs.constants.W_OK))
|
|
103
|
+
return true;
|
|
104
|
+
let entries;
|
|
105
|
+
try {
|
|
106
|
+
entries = (await fsPromises.readdir(dir, { withFileTypes: true }));
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
// An unlistable folder's entries are out of reach only if it cannot be searched.
|
|
110
|
+
return hasAccess(dir, fs.constants.X_OK);
|
|
111
|
+
}
|
|
112
|
+
for (const entry of entries) {
|
|
113
|
+
if (isRoot && entry.name === LOCK_DIR_NAME)
|
|
114
|
+
continue;
|
|
115
|
+
const entryPath = path.join(dir, entry.name);
|
|
116
|
+
if (entry.isDirectory()) {
|
|
117
|
+
if (await canChangeModels(entryPath, false))
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
else if (entry.isSymbolicLink() || entry.name.endsWith('.part')) {
|
|
121
|
+
if (await hasAccess(entryPath, fs.constants.W_OK))
|
|
122
|
+
return true;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
function hasAccess(target, mode) {
|
|
128
|
+
return fsPromises.access(target, mode).then(() => true, () => false);
|
|
129
|
+
}
|
|
130
|
+
// False while another holder has the lock. A filesystem without lock support
|
|
131
|
+
// counts as granted.
|
|
132
|
+
function tryLock(lock, resourceKey) {
|
|
133
|
+
try {
|
|
134
|
+
return fsx.tryLock(lock.fd);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
logger.warn(`⚠️ Cannot lock ${resourceKey} in the models directory; continuing without cross-process coordination`, { error });
|
|
138
|
+
return true;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Hold the cross-process lock for one cached resource until disposed, waiting
|
|
143
|
+
* while another holder has it. The OS releases it if the holder exits.
|
|
144
|
+
* Cancelling while waiting rejects with `DownloadCancelledError`.
|
|
145
|
+
*/
|
|
146
|
+
export async function acquireDownloadLock(resourceKey, signal) {
|
|
147
|
+
if (signal.aborted)
|
|
148
|
+
throw new DownloadCancelledError();
|
|
149
|
+
const lock = await openLockFile(resourceKey);
|
|
150
|
+
if (!lock)
|
|
151
|
+
return { [Symbol.asyncDispose]: async () => { } };
|
|
152
|
+
try {
|
|
153
|
+
if (!tryLock(lock, resourceKey)) {
|
|
154
|
+
logger.info(`⏳ ${resourceKey} is being downloaded elsewhere; waiting for it to finish`);
|
|
155
|
+
do {
|
|
156
|
+
await delay(LOCK_POLL_MS, signal);
|
|
157
|
+
if (signal.aborted)
|
|
158
|
+
throw new DownloadCancelledError();
|
|
159
|
+
} while (!tryLock(lock, resourceKey));
|
|
160
|
+
logger.info(`📥 The other download of ${resourceKey} finished; continuing`);
|
|
161
|
+
}
|
|
162
|
+
return trackHeldHere(resourceKey, lock);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
await lock[Symbol.asyncDispose]();
|
|
166
|
+
throw error;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/** Take every key's lock without waiting, or none; `null` while any is held. */
|
|
170
|
+
export async function tryAcquireDownloadLocks(resourceKeys) {
|
|
171
|
+
const held = [];
|
|
172
|
+
const releaseAll = async () => {
|
|
173
|
+
for (const lock of held)
|
|
174
|
+
await lock[Symbol.asyncDispose]();
|
|
175
|
+
};
|
|
176
|
+
try {
|
|
177
|
+
for (const resourceKey of resourceKeys) {
|
|
178
|
+
const lock = await openLockFile(resourceKey);
|
|
179
|
+
if (!lock)
|
|
180
|
+
continue;
|
|
181
|
+
if (!tryLock(lock, resourceKey)) {
|
|
182
|
+
await lock[Symbol.asyncDispose]();
|
|
183
|
+
await releaseAll();
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
held.push(trackHeldHere(resourceKey, lock));
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
await releaseAll();
|
|
191
|
+
throw error;
|
|
192
|
+
}
|
|
193
|
+
return { [Symbol.asyncDispose]: releaseAll };
|
|
194
|
+
}
|
|
195
|
+
function delay(ms, signal) {
|
|
196
|
+
return new Promise((resolve) => {
|
|
197
|
+
const done = () => {
|
|
198
|
+
clearTimeout(timer);
|
|
199
|
+
signal.removeEventListener('abort', done);
|
|
200
|
+
resolve();
|
|
201
|
+
};
|
|
202
|
+
const timer = setTimeout(done, ms);
|
|
203
|
+
signal.addEventListener('abort', done, { once: true });
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
/** Where a download is written until it is complete and verified. */
|
|
207
|
+
export function partialDownloadPath(finalPath) {
|
|
208
|
+
return `${finalPath}.part`;
|
|
209
|
+
}
|
|
210
|
+
/** Atomically move a verified download into place. */
|
|
211
|
+
export async function publishDownload(finalPath) {
|
|
212
|
+
await renameWithCachedChecksum(partialDownloadPath(finalPath), finalPath);
|
|
213
|
+
}
|
|
@@ -23,6 +23,7 @@ export interface SubscriberRequestBinding {
|
|
|
23
23
|
signal: AbortSignal;
|
|
24
24
|
scope: DisposableScope;
|
|
25
25
|
requestId: string;
|
|
26
|
+
sessionId: string;
|
|
26
27
|
}
|
|
27
28
|
export interface Subscriber {
|
|
28
29
|
id: string;
|
|
@@ -33,13 +34,13 @@ export interface Subscriber {
|
|
|
33
34
|
promise: Promise<string>;
|
|
34
35
|
/** Identity of the registry request this subscriber belongs to, if any. */
|
|
35
36
|
requestId?: string | undefined;
|
|
37
|
+
sessionId?: string | undefined;
|
|
36
38
|
}
|
|
37
39
|
export interface Transfer {
|
|
38
40
|
downloadKey: string;
|
|
39
41
|
abortController: AbortController;
|
|
40
42
|
subscribers: Map<string, Subscriber>;
|
|
41
43
|
lastProgress?: ModelProgressUpdate | undefined;
|
|
42
|
-
downloadPromise?: Promise<string> | undefined;
|
|
43
44
|
clearCache: boolean;
|
|
44
45
|
cacheHit?: boolean;
|
|
45
46
|
}
|
|
@@ -85,7 +86,14 @@ export declare function startOrJoinDownload(downloadKey: string, startDownload:
|
|
|
85
86
|
* asks for clearCache, the partial file is deleted when the last
|
|
86
87
|
* subscriber leaves.
|
|
87
88
|
*/
|
|
88
|
-
export declare function markClearCacheForRequest(requestId: string): boolean;
|
|
89
|
+
export declare function markClearCacheForRequest(requestId: string, sessionId: string): boolean;
|
|
90
|
+
/**
|
|
91
|
+
* Runs `fn` while new downloads and loads writing `resourceKey` are refused, so
|
|
92
|
+
* a cache deletion cannot remove files another call is writing or loading.
|
|
93
|
+
*/
|
|
94
|
+
export declare function withDownloadBlocked<T>(resourceKeys: string[], fn: () => Promise<T>): Promise<T>;
|
|
95
|
+
export declare function hasActiveTransfer(resourceKey: string): boolean;
|
|
96
|
+
export declare function createRegistryCacheKey(registryPath: string): string;
|
|
89
97
|
export declare function cancelTransfer(downloadKey: string, clearCache?: boolean): void;
|
|
90
98
|
export declare function createHyperdriveDownloadKey(hyperdriveKey: string, modelFileName: string): string;
|
|
91
99
|
export declare function createHttpDownloadKey(url: string): string;
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { AbortController } from 'bare-abort-controller';
|
|
2
|
-
import { DownloadCancelledError, InferenceCancelledError } from '../../errors/index.js';
|
|
2
|
+
import { DeleteCacheFailedError, DownloadCancelledError, InferenceCancelledError, ModelLoadFailedError } from '../../errors/index.js';
|
|
3
3
|
import { getEngineLogger } from '../../logging/index.js';
|
|
4
4
|
import { getRequestRegistry } from '../../runtime/index.js';
|
|
5
5
|
const logger = getEngineLogger();
|
|
6
6
|
const activeTransfers = new Map();
|
|
7
|
+
// Every download until it settles, including a cancelled one a retry replaced.
|
|
8
|
+
const runningDownloads = new Set();
|
|
9
|
+
const deletingKeys = new Set();
|
|
7
10
|
let nextSubscriberId = 0;
|
|
8
|
-
function createSubscriber(onProgress,
|
|
11
|
+
function createSubscriber(onProgress, request) {
|
|
9
12
|
let resolve;
|
|
10
13
|
let reject;
|
|
11
14
|
const promise = new Promise((res, rej) => {
|
|
@@ -19,7 +22,8 @@ function createSubscriber(onProgress, requestId) {
|
|
|
19
22
|
resolve,
|
|
20
23
|
reject,
|
|
21
24
|
promise,
|
|
22
|
-
requestId
|
|
25
|
+
requestId: request?.requestId,
|
|
26
|
+
sessionId: request?.sessionId
|
|
23
27
|
};
|
|
24
28
|
}
|
|
25
29
|
function settleSubscriber(subscriber, result) {
|
|
@@ -118,10 +122,17 @@ function attachRequestBinding(transfer, subscriber, request) {
|
|
|
118
122
|
});
|
|
119
123
|
}
|
|
120
124
|
export function startOrJoinDownload(downloadKey, startDownload, onProgress, request) {
|
|
125
|
+
if (deletingKeys.has(getCacheResourceKey(downloadKey))) {
|
|
126
|
+
return {
|
|
127
|
+
promise: Promise.reject(new ModelLoadFailedError(`The cache for ${downloadKey} is being deleted`)),
|
|
128
|
+
joined: false,
|
|
129
|
+
getCacheHit: () => undefined
|
|
130
|
+
};
|
|
131
|
+
}
|
|
121
132
|
const existing = activeTransfers.get(downloadKey);
|
|
122
133
|
if (existing && !existing.abortController.signal.aborted) {
|
|
123
134
|
logger.info(`📥 Reusing existing download for: ${downloadKey}`);
|
|
124
|
-
const subscriber = createSubscriber(onProgress, request
|
|
135
|
+
const subscriber = createSubscriber(onProgress, request);
|
|
125
136
|
existing.subscribers.set(subscriber.id, subscriber);
|
|
126
137
|
if (request) {
|
|
127
138
|
attachRequestBinding(existing, subscriber, request);
|
|
@@ -142,7 +153,7 @@ export function startOrJoinDownload(downloadKey, startDownload, onProgress, requ
|
|
|
142
153
|
subscribers: new Map(),
|
|
143
154
|
clearCache: false
|
|
144
155
|
};
|
|
145
|
-
const initialSubscriber = createSubscriber(onProgress, request
|
|
156
|
+
const initialSubscriber = createSubscriber(onProgress, request);
|
|
146
157
|
transfer.subscribers.set(initialSubscriber.id, initialSubscriber);
|
|
147
158
|
activeTransfers.set(downloadKey, transfer);
|
|
148
159
|
if (request) {
|
|
@@ -158,7 +169,7 @@ export function startOrJoinDownload(downloadKey, startDownload, onProgress, requ
|
|
|
158
169
|
transfer.cacheHit = cacheHit;
|
|
159
170
|
}
|
|
160
171
|
});
|
|
161
|
-
|
|
172
|
+
runningDownloads.add(downloadPromise);
|
|
162
173
|
downloadPromise
|
|
163
174
|
.then((path) => {
|
|
164
175
|
for (const sub of transfer.subscribers.values()) {
|
|
@@ -171,6 +182,7 @@ export function startOrJoinDownload(downloadKey, startDownload, onProgress, requ
|
|
|
171
182
|
}
|
|
172
183
|
})
|
|
173
184
|
.finally(() => {
|
|
185
|
+
runningDownloads.delete(downloadPromise);
|
|
174
186
|
if (activeTransfers.get(downloadKey) === transfer) {
|
|
175
187
|
activeTransfers.delete(downloadKey);
|
|
176
188
|
}
|
|
@@ -211,10 +223,10 @@ export function startOrJoinDownload(downloadKey, startDownload, onProgress, requ
|
|
|
211
223
|
* asks for clearCache, the partial file is deleted when the last
|
|
212
224
|
* subscriber leaves.
|
|
213
225
|
*/
|
|
214
|
-
export function markClearCacheForRequest(requestId) {
|
|
226
|
+
export function markClearCacheForRequest(requestId, sessionId) {
|
|
215
227
|
for (const transfer of activeTransfers.values()) {
|
|
216
228
|
for (const sub of transfer.subscribers.values()) {
|
|
217
|
-
if (sub.requestId === requestId) {
|
|
229
|
+
if (sub.requestId === requestId && sub.sessionId === sessionId) {
|
|
218
230
|
transfer.clearCache = true;
|
|
219
231
|
return true;
|
|
220
232
|
}
|
|
@@ -222,6 +234,43 @@ export function markClearCacheForRequest(requestId) {
|
|
|
222
234
|
}
|
|
223
235
|
return false;
|
|
224
236
|
}
|
|
237
|
+
/**
|
|
238
|
+
* Runs `fn` while new downloads and loads writing `resourceKey` are refused, so
|
|
239
|
+
* a cache deletion cannot remove files another call is writing or loading.
|
|
240
|
+
*/
|
|
241
|
+
export async function withDownloadBlocked(resourceKeys, fn) {
|
|
242
|
+
const busy = resourceKeys.find((key) => deletingKeys.has(key));
|
|
243
|
+
if (busy) {
|
|
244
|
+
throw new DeleteCacheFailedError(`The cache for ${busy} is already being deleted`);
|
|
245
|
+
}
|
|
246
|
+
for (const key of resourceKeys)
|
|
247
|
+
deletingKeys.add(key);
|
|
248
|
+
try {
|
|
249
|
+
return await fn();
|
|
250
|
+
}
|
|
251
|
+
finally {
|
|
252
|
+
for (const key of resourceKeys)
|
|
253
|
+
deletingKeys.delete(key);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
export function hasActiveTransfer(resourceKey) {
|
|
257
|
+
for (const [downloadKey, transfer] of activeTransfers) {
|
|
258
|
+
if (getCacheResourceKey(downloadKey) !== resourceKey)
|
|
259
|
+
continue;
|
|
260
|
+
if (!transfer.abortController.signal.aborted)
|
|
261
|
+
return true;
|
|
262
|
+
}
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
// The registry cache is laid out by path alone, so every source of a path
|
|
266
|
+
// writes the same files.
|
|
267
|
+
export function createRegistryCacheKey(registryPath) {
|
|
268
|
+
return `registry-cache:${registryPath}`;
|
|
269
|
+
}
|
|
270
|
+
function getCacheResourceKey(downloadKey) {
|
|
271
|
+
const registry = /^registry:[^:]*:(.*)$/.exec(downloadKey);
|
|
272
|
+
return registry ? createRegistryCacheKey(registry[1]) : downloadKey;
|
|
273
|
+
}
|
|
225
274
|
export function cancelTransfer(downloadKey, clearCache = false) {
|
|
226
275
|
const transfer = activeTransfers.get(downloadKey);
|
|
227
276
|
if (!transfer)
|
|
@@ -233,6 +282,7 @@ export function cancelTransfer(downloadKey, clearCache = false) {
|
|
|
233
282
|
if (sub.requestId !== undefined) {
|
|
234
283
|
registry.cancel({
|
|
235
284
|
requestId: sub.requestId,
|
|
285
|
+
sessionId: sub.sessionId,
|
|
236
286
|
reason: 'download-transfer-cancel'
|
|
237
287
|
});
|
|
238
288
|
}
|
|
@@ -292,9 +342,7 @@ export async function cleanupDownloads() {
|
|
|
292
342
|
return;
|
|
293
343
|
isCleaningUp = true;
|
|
294
344
|
try {
|
|
295
|
-
const downloadPromises = Array.from(
|
|
296
|
-
.filter((t) => t.downloadPromise !== undefined)
|
|
297
|
-
.map((t) => t.downloadPromise.catch(() => { }));
|
|
345
|
+
const downloadPromises = Array.from(runningDownloads, (download) => download.catch(() => { }));
|
|
298
346
|
cancelAllDownloads();
|
|
299
347
|
if (downloadPromises.length > 0) {
|
|
300
348
|
await Promise.allSettled(downloadPromises);
|
|
@@ -303,4 +351,7 @@ export async function cleanupDownloads() {
|
|
|
303
351
|
catch (error) {
|
|
304
352
|
logger.error('❌ Error during download cleanup:', error);
|
|
305
353
|
}
|
|
354
|
+
finally {
|
|
355
|
+
isCleaningUp = false;
|
|
356
|
+
}
|
|
306
357
|
}
|
|
@@ -52,18 +52,20 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
52
52
|
});
|
|
53
53
|
import { normalizeModelType, PROFILING_KEY, OPERATION_EVENT_KEY } from '../../schemas/index.js';
|
|
54
54
|
import { loadModel } from '../../plugins/ops/load-model.js';
|
|
55
|
+
import { withModelWorkLock } from '../../runtime/model-work-lock.js';
|
|
55
56
|
import { createResolveSession } from '../../handlers/load-model/resolve-session.js';
|
|
56
57
|
import { nowMs, generateProfileId } from '../../profiling/clock.js';
|
|
57
58
|
import { getModelEntry, updateModelConfig } from '../../runtime/model-registry.js';
|
|
58
59
|
import { generateShortHash, canonicalConfigString, transformConfigForReload } from '../../utils/index.js';
|
|
59
60
|
import { buildDownloadProfilingFields } from '../../handlers/load-model/types.js';
|
|
60
61
|
import { readBackendDiagnostics } from '../../profiling/backend-diagnostics.js';
|
|
61
|
-
import { ConfigReloadNotSupportedError, InferenceCancelledError, ModelTypeMismatchError, ModelNotFoundError, ModelLoadFailedError, PluginLoadConfigValidationFailedError, PluginNotFoundError } from '../../errors/index.js';
|
|
62
|
+
import { ConfigReloadNotSupportedError, InferenceCancelledError, ModelLeasedError, ModelReconfiguredError, ModelTypeMismatchError, ModelNotFoundError, ModelLoadFailedError, ModelRegistryBusyError, PluginLoadConfigValidationFailedError, PluginNotFoundError } from '../../errors/index.js';
|
|
62
63
|
import { getEngineLogger } from '../../logging/index.js';
|
|
63
64
|
import { formatZodError } from '../../utils/zod-error.js';
|
|
64
65
|
import { getPlugin } from '../../plugins/index.js';
|
|
65
66
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
66
67
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
68
|
+
import { resolveSessionId } from '../../runtime/session-id.js';
|
|
67
69
|
const logger = getEngineLogger();
|
|
68
70
|
// ---------------------------------------------------------------------------
|
|
69
71
|
// Handler
|
|
@@ -79,6 +81,7 @@ export async function handleLoadModel(request, progressCallback) {
|
|
|
79
81
|
const profilingMeta = request[PROFILING_KEY];
|
|
80
82
|
const profilingEnabled = profilingMeta?.enabled !== false && !!profilingMeta;
|
|
81
83
|
const requestId = request.requestId ?? generateRandomRequestId();
|
|
84
|
+
const sessionId = resolveSessionId(request);
|
|
82
85
|
// The handler `modelId` is derived from the config hash below — it
|
|
83
86
|
// isn't known until after `resolveConfig` runs. The registry context
|
|
84
87
|
// is opened with `modelId: undefined` so a cancel-by-modelId fired
|
|
@@ -86,6 +89,7 @@ export async function handleLoadModel(request, progressCallback) {
|
|
|
86
89
|
// half-built entry). Cancel-by-`requestId` works from `begin(...)` on.
|
|
87
90
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
88
91
|
requestId,
|
|
92
|
+
sessionId,
|
|
89
93
|
kind: 'loadModel'
|
|
90
94
|
}), true);
|
|
91
95
|
const log = withRequestContext(getEngineLogger(), ctx);
|
|
@@ -110,7 +114,8 @@ export async function handleLoadModel(request, progressCallback) {
|
|
|
110
114
|
requestBinding: {
|
|
111
115
|
signal: ctx.signal,
|
|
112
116
|
scope: ctx.scope,
|
|
113
|
-
requestId
|
|
117
|
+
requestId,
|
|
118
|
+
sessionId
|
|
114
119
|
},
|
|
115
120
|
security: { requireHttpChecksum, requireSecureTransport }
|
|
116
121
|
});
|
|
@@ -169,7 +174,8 @@ export async function handleLoadModel(request, progressCallback) {
|
|
|
169
174
|
modelName
|
|
170
175
|
}, {
|
|
171
176
|
...(profilingEnabled && { collectTiming: true }),
|
|
172
|
-
request: { signal: ctx.signal, requestId }
|
|
177
|
+
request: { signal: ctx.signal, requestId },
|
|
178
|
+
sessionId
|
|
173
179
|
});
|
|
174
180
|
// Load phase soft-cancel limitation: `plugin.createModel(...)` and
|
|
175
181
|
// `model.load(false)` ran without a signal. A cancel that arrived
|
|
@@ -226,6 +232,11 @@ export async function handleLoadModel(request, progressCallback) {
|
|
|
226
232
|
log.info(`loadModel cancelled requestId=${requestId}`);
|
|
227
233
|
throw error;
|
|
228
234
|
}
|
|
235
|
+
if (error instanceof ModelReconfiguredError)
|
|
236
|
+
throw error;
|
|
237
|
+
// Typed so callers can tell another process has the registry.
|
|
238
|
+
if (error instanceof ModelRegistryBusyError)
|
|
239
|
+
throw error;
|
|
229
240
|
logger.error('Error loading model:', error);
|
|
230
241
|
return {
|
|
231
242
|
type: 'loadModel',
|
|
@@ -245,43 +256,85 @@ export async function handleLoadModel(request, progressCallback) {
|
|
|
245
256
|
}
|
|
246
257
|
}
|
|
247
258
|
async function handleConfigReload(request) {
|
|
248
|
-
const {
|
|
259
|
+
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
249
260
|
try {
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
261
|
+
// Registered like a load, so a cancel by id reaches a reload still waiting
|
|
262
|
+
// for the lock, and a session close waits for a running reload before it
|
|
263
|
+
// releases the session's leases.
|
|
264
|
+
const requestId = request.requestId ?? generateRandomRequestId();
|
|
265
|
+
const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
|
|
266
|
+
requestId,
|
|
267
|
+
sessionId: resolveSessionId(request),
|
|
268
|
+
kind: 'loadModel'
|
|
269
|
+
}), true);
|
|
270
|
+
try {
|
|
271
|
+
// Under the same lock as a load of this id: a session loading the same
|
|
272
|
+
// configuration waits until the reload is done instead of taking a lease
|
|
273
|
+
// on a model whose configuration is changing.
|
|
274
|
+
return await withModelWorkLock(request.modelId, () => {
|
|
275
|
+
if (ctx.signal.aborted)
|
|
276
|
+
throw new InferenceCancelledError(requestId);
|
|
277
|
+
return reloadConfig(request);
|
|
278
|
+
}, { signal: ctx.signal, requestId });
|
|
258
279
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
280
|
+
catch (error) {
|
|
281
|
+
// A refused reload and a cancel are typed errors the client reacts to,
|
|
282
|
+
// not a generic `success: false` envelope.
|
|
283
|
+
if (error instanceof ModelLeasedError || error instanceof InferenceCancelledError)
|
|
284
|
+
throw error;
|
|
285
|
+
logger.error('Error reloading config:', error);
|
|
286
|
+
return {
|
|
287
|
+
type: 'loadModel',
|
|
288
|
+
success: false,
|
|
289
|
+
error: error instanceof Error ? error.message : 'Unknown error'
|
|
290
|
+
};
|
|
263
291
|
}
|
|
264
|
-
const mergedConfig = {
|
|
265
|
-
...currentConfig,
|
|
266
|
-
...modelConfig
|
|
267
|
-
};
|
|
268
|
-
const reloadConfig = transformConfigForReload(storedModelType, mergedConfig);
|
|
269
|
-
await model.reload(reloadConfig);
|
|
270
|
-
updateModelConfig(modelId, mergedConfig);
|
|
271
|
-
return {
|
|
272
|
-
type: 'loadModel',
|
|
273
|
-
success: true,
|
|
274
|
-
modelId
|
|
275
|
-
};
|
|
276
292
|
}
|
|
277
|
-
catch (
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
type: 'loadModel',
|
|
281
|
-
success: false,
|
|
282
|
-
error: error instanceof Error ? error.message : 'Unknown error'
|
|
283
|
-
};
|
|
293
|
+
catch (e_2) {
|
|
294
|
+
env_2.error = e_2;
|
|
295
|
+
env_2.hasError = true;
|
|
284
296
|
}
|
|
297
|
+
finally {
|
|
298
|
+
const result_2 = __disposeResources(env_2);
|
|
299
|
+
if (result_2)
|
|
300
|
+
await result_2;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
async function reloadConfig(request) {
|
|
304
|
+
const { modelId, modelType, modelConfig } = request;
|
|
305
|
+
const entry = getModelEntry(modelId);
|
|
306
|
+
if (!entry) {
|
|
307
|
+
throw new ModelNotFoundError(modelId);
|
|
308
|
+
}
|
|
309
|
+
const storedModelType = entry.local.modelType;
|
|
310
|
+
const normalizedRequestType = normalizeModelType(modelType);
|
|
311
|
+
if (storedModelType !== normalizedRequestType) {
|
|
312
|
+
throw new ModelTypeMismatchError(storedModelType, normalizedRequestType);
|
|
313
|
+
}
|
|
314
|
+
const model = entry.local.model;
|
|
315
|
+
const currentConfig = entry.local.config;
|
|
316
|
+
if (typeof model.reload !== 'function') {
|
|
317
|
+
throw new ConfigReloadNotSupportedError(modelId);
|
|
318
|
+
}
|
|
319
|
+
// A reload mutates the one shared native model, so it is refused while
|
|
320
|
+
// any other session depends on the current configuration.
|
|
321
|
+
const sessionId = resolveSessionId(request);
|
|
322
|
+
const otherLeases = Array.from(entry.leases).filter((id) => id !== sessionId).length;
|
|
323
|
+
if (otherLeases > 0) {
|
|
324
|
+
throw new ModelLeasedError(modelId, otherLeases);
|
|
325
|
+
}
|
|
326
|
+
const mergedConfig = {
|
|
327
|
+
...currentConfig,
|
|
328
|
+
...modelConfig
|
|
329
|
+
};
|
|
330
|
+
const nativeConfig = transformConfigForReload(storedModelType, mergedConfig);
|
|
331
|
+
await model.reload(nativeConfig);
|
|
332
|
+
updateModelConfig(modelId, mergedConfig);
|
|
333
|
+
return {
|
|
334
|
+
type: 'loadModel',
|
|
335
|
+
success: true,
|
|
336
|
+
modelId
|
|
337
|
+
};
|
|
285
338
|
}
|
|
286
339
|
function isReloadConfigRequest(request) {
|
|
287
340
|
return 'modelId' in request && !('modelSrc' in request);
|