@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -41,6 +41,8 @@ export interface DownloadRequestBinding {
|
|
|
41
41
|
signal: AbortSignal;
|
|
42
42
|
scope: DisposableScope;
|
|
43
43
|
requestId: string;
|
|
44
|
+
/** Session the request runs as; the registry resolves `requestId` within it. */
|
|
45
|
+
sessionId: string;
|
|
44
46
|
}
|
|
45
47
|
export interface DownloadHooks {
|
|
46
48
|
onDownloadKey?: (key: string) => void;
|
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
import type { LoggingStreamRequest, LoggingStreamResponse } from '../schemas/index.js';
|
|
2
|
-
|
|
2
|
+
import type { StreamHandlerContext } from '../handlers/types.js';
|
|
3
|
+
export declare function handleLoggingStream(request: LoggingStreamRequest, context?: StreamHandlerContext): AsyncGenerator<LoggingStreamResponse>;
|
|
@@ -1,8 +1,27 @@
|
|
|
1
1
|
import { registerLoggingStream, unregisterLoggingStream } from '../runtime/logging-stream-registry.js';
|
|
2
|
-
|
|
2
|
+
import { isOwnerSession, isSessionOpen, onSessionClose } from '../runtime/session-registry.js';
|
|
3
|
+
import { resolveSessionId } from '../runtime/session-id.js';
|
|
4
|
+
export async function* handleLoggingStream(request, context = {}) {
|
|
3
5
|
const { id } = request;
|
|
6
|
+
const { signal } = context;
|
|
7
|
+
if (signal?.aborted)
|
|
8
|
+
return;
|
|
9
|
+
// A subscription is not a registered request, so the session close that
|
|
10
|
+
// drains requests would not reach it; it ends on its own close listener.
|
|
11
|
+
const sessionId = resolveSessionId(request);
|
|
12
|
+
const scoped = !isOwnerSession(sessionId);
|
|
13
|
+
if (scoped && !isSessionOpen(sessionId))
|
|
14
|
+
return;
|
|
15
|
+
let stopped = false;
|
|
4
16
|
const logQueue = [];
|
|
5
17
|
let pendingResolve = null;
|
|
18
|
+
const wake = () => {
|
|
19
|
+
if (pendingResolve) {
|
|
20
|
+
const resolve = pendingResolve;
|
|
21
|
+
pendingResolve = null;
|
|
22
|
+
resolve();
|
|
23
|
+
}
|
|
24
|
+
};
|
|
6
25
|
const streamHandler = (level, namespace, message, sourceId) => {
|
|
7
26
|
const logResponse = {
|
|
8
27
|
type: 'loggingStream',
|
|
@@ -15,25 +34,34 @@ export async function* handleLoggingStream(request) {
|
|
|
15
34
|
timestamp: Date.now()
|
|
16
35
|
};
|
|
17
36
|
logQueue.push(logResponse);
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
37
|
+
wake();
|
|
38
|
+
};
|
|
39
|
+
// Release at once, even if nobody is reading; a quiet id would wait forever.
|
|
40
|
+
const stop = () => {
|
|
41
|
+
stopped = true;
|
|
42
|
+
unregisterLoggingStream(id, streamHandler);
|
|
43
|
+
logQueue.length = 0;
|
|
44
|
+
wake();
|
|
23
45
|
};
|
|
24
46
|
registerLoggingStream(id, streamHandler);
|
|
47
|
+
signal?.addEventListener('abort', stop, { once: true });
|
|
48
|
+
const removeCloseListener = scoped ? onSessionClose(sessionId, stop) : undefined;
|
|
25
49
|
try {
|
|
26
50
|
while (true) {
|
|
27
51
|
while (logQueue.length > 0) {
|
|
28
52
|
yield logQueue.shift();
|
|
29
53
|
}
|
|
30
|
-
//
|
|
54
|
+
// Stopped during a read: nothing to wake, so end here.
|
|
55
|
+
if (stopped || signal?.aborted)
|
|
56
|
+
return;
|
|
31
57
|
await new Promise((resolve) => {
|
|
32
58
|
pendingResolve = resolve;
|
|
33
59
|
});
|
|
34
60
|
}
|
|
35
61
|
}
|
|
36
62
|
finally {
|
|
63
|
+
signal?.removeEventListener('abort', stop);
|
|
64
|
+
removeCloseListener?.();
|
|
37
65
|
unregisterLoggingStream(id, streamHandler);
|
|
38
66
|
}
|
|
39
67
|
}
|
|
@@ -1,7 +1,62 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
1
53
|
import { getModelEntry } from '../runtime/model-registry.js';
|
|
2
54
|
import { getPlugin, getPluginHandler } from '../plugins/index.js';
|
|
55
|
+
import { getRequestRegistry } from '../runtime/index.js';
|
|
56
|
+
import { OWNER_SESSION_ID } from '../runtime/session-id.js';
|
|
57
|
+
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
3
58
|
import { profileReplyHandler, profileStreamHandler } from '../profiling/index.js';
|
|
4
|
-
import { PluginNotFoundError, PluginHandlerNotFoundError, PluginHandlerTypeMismatchError, PluginRequestValidationFailedError, PluginResponseValidationFailedError, ModelNotFoundError } from '../errors/index.js';
|
|
59
|
+
import { InferenceCancelledError, PluginNotFoundError, PluginHandlerNotFoundError, PluginHandlerTypeMismatchError, PluginRequestValidationFailedError, PluginResponseValidationFailedError, ModelNotFoundError } from '../errors/index.js';
|
|
5
60
|
import { getEngineLogger } from '../logging/index.js';
|
|
6
61
|
import { formatZodError } from '../utils/zod-error.js';
|
|
7
62
|
const logger = getEngineLogger();
|
|
@@ -22,57 +77,115 @@ function resolvePluginHandler(modelId, handlerName) {
|
|
|
22
77
|
}
|
|
23
78
|
return { modelType, plugin, handlerDef };
|
|
24
79
|
}
|
|
80
|
+
// Every invoke is a registry request of kind `plugin` on the model, so a
|
|
81
|
+
// caller can cancel it by id and a session close or model unload sweeps it.
|
|
82
|
+
// The handler receives the abort signal through its context argument.
|
|
83
|
+
function beginInvoke(request) {
|
|
84
|
+
return getRequestRegistry().begin({
|
|
85
|
+
requestId: request.requestId ?? generateRandomRequestId(),
|
|
86
|
+
sessionId: request.sessionId,
|
|
87
|
+
kind: 'plugin',
|
|
88
|
+
modelId: request.modelId
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
// The owner's session is passed as undefined, the way the wire names it, so
|
|
92
|
+
// an inner request begun from the context registers like a direct one.
|
|
93
|
+
function invokeContext(ctx) {
|
|
94
|
+
return {
|
|
95
|
+
signal: ctx.signal,
|
|
96
|
+
requestId: ctx.requestId,
|
|
97
|
+
sessionId: ctx.sessionId === OWNER_SESSION_ID ? undefined : ctx.sessionId
|
|
98
|
+
};
|
|
99
|
+
}
|
|
25
100
|
export async function handlePluginInvoke(request) {
|
|
26
101
|
return profileReplyHandler({ op: 'pluginInvoke', request }, async () => {
|
|
27
|
-
const {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
102
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
103
|
+
try {
|
|
104
|
+
const { modelId, handler: handlerName, params } = request;
|
|
105
|
+
logger.debug(`[pluginInvoke] modelId=${modelId} handler=${handlerName}`);
|
|
106
|
+
const { handlerDef } = resolvePluginHandler(modelId, handlerName);
|
|
107
|
+
if (handlerDef.streaming) {
|
|
108
|
+
throw new PluginHandlerTypeMismatchError(handlerName, 'reply', 'streaming');
|
|
109
|
+
}
|
|
110
|
+
const parseResult = handlerDef.requestSchema.safeParse(params);
|
|
111
|
+
if (!parseResult.success) {
|
|
112
|
+
throw new PluginRequestValidationFailedError(handlerName, formatZodError(parseResult.error));
|
|
113
|
+
}
|
|
114
|
+
const ctx = __addDisposableResource(env_1, await beginInvoke(request), true);
|
|
115
|
+
if (ctx.signal.aborted)
|
|
116
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
117
|
+
const result = await handlerDef.handler(parseResult.data, undefined, invokeContext(ctx));
|
|
118
|
+
if (ctx.signal.aborted)
|
|
119
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
120
|
+
const responseParseResult = handlerDef.responseSchema.safeParse(result);
|
|
121
|
+
if (!responseParseResult.success) {
|
|
122
|
+
throw new PluginResponseValidationFailedError(handlerName, formatZodError(responseParseResult.error));
|
|
123
|
+
}
|
|
124
|
+
return {
|
|
125
|
+
type: 'pluginInvoke',
|
|
126
|
+
requestId: ctx.requestId,
|
|
127
|
+
result: responseParseResult.data
|
|
128
|
+
};
|
|
32
129
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
130
|
+
catch (e_1) {
|
|
131
|
+
env_1.error = e_1;
|
|
132
|
+
env_1.hasError = true;
|
|
36
133
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
134
|
+
finally {
|
|
135
|
+
const result_1 = __disposeResources(env_1);
|
|
136
|
+
if (result_1)
|
|
137
|
+
await result_1;
|
|
41
138
|
}
|
|
42
|
-
return {
|
|
43
|
-
type: 'pluginInvoke',
|
|
44
|
-
result: responseParseResult.data
|
|
45
|
-
};
|
|
46
139
|
});
|
|
47
140
|
}
|
|
48
141
|
export async function* handlePluginInvokeStream(request) {
|
|
49
142
|
yield* profileStreamHandler({ op: 'pluginInvokeStream', request }, async function* () {
|
|
50
|
-
const {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
for await (const chunk of generator) {
|
|
62
|
-
const responseParseResult = handlerDef.responseSchema.safeParse(chunk);
|
|
63
|
-
if (!responseParseResult.success) {
|
|
64
|
-
throw new PluginResponseValidationFailedError(handlerName, formatZodError(responseParseResult.error));
|
|
143
|
+
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
144
|
+
try {
|
|
145
|
+
const { modelId, handler: handlerName, params } = request;
|
|
146
|
+
logger.debug(`[pluginInvokeStream] modelId=${modelId} handler=${handlerName}`);
|
|
147
|
+
const { handlerDef } = resolvePluginHandler(modelId, handlerName);
|
|
148
|
+
if (!handlerDef.streaming) {
|
|
149
|
+
throw new PluginHandlerTypeMismatchError(handlerName, 'streaming', 'reply');
|
|
150
|
+
}
|
|
151
|
+
const parseResult = handlerDef.requestSchema.safeParse(params);
|
|
152
|
+
if (!parseResult.success) {
|
|
153
|
+
throw new PluginRequestValidationFailedError(handlerName, formatZodError(parseResult.error));
|
|
65
154
|
}
|
|
155
|
+
const ctx = __addDisposableResource(env_2, await beginInvoke(request), true);
|
|
156
|
+
if (ctx.signal.aborted)
|
|
157
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
158
|
+
const generator = handlerDef.handler(parseResult.data, undefined, invokeContext(ctx));
|
|
159
|
+
for await (const chunk of generator) {
|
|
160
|
+
if (ctx.signal.aborted)
|
|
161
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
162
|
+
const responseParseResult = handlerDef.responseSchema.safeParse(chunk);
|
|
163
|
+
if (!responseParseResult.success) {
|
|
164
|
+
throw new PluginResponseValidationFailedError(handlerName, formatZodError(responseParseResult.error));
|
|
165
|
+
}
|
|
166
|
+
yield {
|
|
167
|
+
type: 'pluginInvokeStream',
|
|
168
|
+
result: responseParseResult.data,
|
|
169
|
+
done: false
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
if (ctx.signal.aborted)
|
|
173
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
66
174
|
yield {
|
|
67
175
|
type: 'pluginInvokeStream',
|
|
68
|
-
|
|
69
|
-
|
|
176
|
+
requestId: ctx.requestId,
|
|
177
|
+
result: null,
|
|
178
|
+
done: true
|
|
70
179
|
};
|
|
71
180
|
}
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
181
|
+
catch (e_2) {
|
|
182
|
+
env_2.error = e_2;
|
|
183
|
+
env_2.hasError = true;
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
const result_2 = __disposeResources(env_2);
|
|
187
|
+
if (result_2)
|
|
188
|
+
await result_2;
|
|
189
|
+
}
|
|
77
190
|
});
|
|
78
191
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { ModelRegistryListResponse, ModelRegistrySearchRequest, ModelRegistrySearchResponse, ModelRegistryGetModelRequest, ModelRegistryGetModelResponse, ModelRegistryEntry } from '../schemas/index.js';
|
|
1
|
+
import type { ModelRegistryListRequest, ModelRegistryListResponse, ModelRegistrySearchRequest, ModelRegistrySearchResponse, ModelRegistryGetModelRequest, ModelRegistryGetModelResponse, ModelRegistryEntry } from '../schemas/index.js';
|
|
2
2
|
import type { QVACModelEntry } from '@qvac/registry-client';
|
|
3
3
|
export declare function processRegistryModel(model: QVACModelEntry): ModelRegistryEntry | null;
|
|
4
|
-
export declare function handleModelRegistryList(): Promise<ModelRegistryListResponse>;
|
|
4
|
+
export declare function handleModelRegistryList(request?: ModelRegistryListRequest): Promise<ModelRegistryListResponse>;
|
|
5
5
|
export declare function handleModelRegistrySearch(request: ModelRegistrySearchRequest): Promise<ModelRegistrySearchResponse>;
|
|
6
6
|
export declare function handleModelRegistryGetModel(request: ModelRegistryGetModelRequest): Promise<ModelRegistryGetModelResponse>;
|
|
@@ -1,9 +1,64 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
1
53
|
import Buffer from 'bare-buffer';
|
|
2
54
|
import { REGISTRY_ERROR_CODES } from '../schemas/errors.js';
|
|
3
55
|
import { getAddonFromEngine, resolveCanonicalEngine, resolveRegistryEngine } from '../schemas/engine-addon-map.js';
|
|
4
|
-
import {
|
|
56
|
+
import { acquireRegistryClient } from '../runtime/registry-client.js';
|
|
57
|
+
import { getRequestRegistry } from '../runtime/index.js';
|
|
58
|
+
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
59
|
+
import { resolveSessionId } from '../runtime/session-id.js';
|
|
5
60
|
import { getEngineLogger } from '../logging/index.js';
|
|
6
|
-
import { ModelRegistryQueryFailedError } from '../errors/index.js';
|
|
61
|
+
import { InferenceCancelledError, ModelRegistryBusyError, ModelRegistryModelNotFoundError, ModelRegistryQueryFailedError } from '../errors/index.js';
|
|
7
62
|
const logger = getEngineLogger();
|
|
8
63
|
function toHexString(value) {
|
|
9
64
|
if (!value)
|
|
@@ -17,6 +72,46 @@ function toHexString(value) {
|
|
|
17
72
|
}
|
|
18
73
|
return '';
|
|
19
74
|
}
|
|
75
|
+
// One query, cancellable by `requestId` while it waits for another process to
|
|
76
|
+
// release the registry.
|
|
77
|
+
async function withRegistryClient(request, query) {
|
|
78
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
79
|
+
try {
|
|
80
|
+
const id = request.requestId ?? generateRandomRequestId();
|
|
81
|
+
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
82
|
+
requestId: id,
|
|
83
|
+
sessionId: resolveSessionId(request),
|
|
84
|
+
kind: 'modelRegistry'
|
|
85
|
+
}), true);
|
|
86
|
+
const lease = __addDisposableResource(env_1, await acquireRegistryClient({
|
|
87
|
+
wait: 'query',
|
|
88
|
+
signal: ctx.signal,
|
|
89
|
+
cancelled: () => new InferenceCancelledError(id)
|
|
90
|
+
})
|
|
91
|
+
// Awaited here: returning the promise would release the lease before the query ends.
|
|
92
|
+
, true);
|
|
93
|
+
// Awaited here: returning the promise would release the lease before the query ends.
|
|
94
|
+
return await query(lease.client);
|
|
95
|
+
}
|
|
96
|
+
catch (e_1) {
|
|
97
|
+
env_1.error = e_1;
|
|
98
|
+
env_1.hasError = true;
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
const result_1 = __disposeResources(env_1);
|
|
102
|
+
if (result_1)
|
|
103
|
+
await result_1;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// Errors a caller can act on pass through unwrapped.
|
|
107
|
+
function isPassThrough(error) {
|
|
108
|
+
const code = error.code;
|
|
109
|
+
return (error instanceof ModelRegistryBusyError ||
|
|
110
|
+
error instanceof InferenceCancelledError ||
|
|
111
|
+
code === REGISTRY_ERROR_CODES.FAILED_TO_CONNECT ||
|
|
112
|
+
code === REGISTRY_ERROR_CODES.FAILED_TO_CLOSE ||
|
|
113
|
+
code === REGISTRY_ERROR_CODES.MODEL_NOT_FOUND);
|
|
114
|
+
}
|
|
20
115
|
export function processRegistryModel(model) {
|
|
21
116
|
const engine = resolveRegistryEngine(model.engine || '', model.tags);
|
|
22
117
|
if (!engine) {
|
|
@@ -52,11 +147,10 @@ export function processRegistryModel(model) {
|
|
|
52
147
|
params: model.params || ''
|
|
53
148
|
};
|
|
54
149
|
}
|
|
55
|
-
export async function handleModelRegistryList() {
|
|
150
|
+
export async function handleModelRegistryList(request = { type: 'modelRegistryList' }) {
|
|
56
151
|
logger.debug('Handling QVAC model registry list request');
|
|
57
152
|
try {
|
|
58
|
-
const
|
|
59
|
-
const registryModels = await client.findBy();
|
|
153
|
+
const registryModels = await withRegistryClient(request, (client) => client.findBy());
|
|
60
154
|
const models = registryModels
|
|
61
155
|
.map(processRegistryModel)
|
|
62
156
|
.filter((m) => m !== null);
|
|
@@ -69,13 +163,8 @@ export async function handleModelRegistryList() {
|
|
|
69
163
|
}
|
|
70
164
|
catch (error) {
|
|
71
165
|
logger.error('QVAC model registry list failed:', error);
|
|
72
|
-
|
|
73
|
-
const qvacError = error;
|
|
74
|
-
if (qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CONNECT ||
|
|
75
|
-
qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CLOSE ||
|
|
76
|
-
qvacError.code === REGISTRY_ERROR_CODES.MODEL_NOT_FOUND) {
|
|
166
|
+
if (isPassThrough(error))
|
|
77
167
|
throw error;
|
|
78
|
-
}
|
|
79
168
|
// Wrap unknown errors in a typed error
|
|
80
169
|
throw new ModelRegistryQueryFailedError(error instanceof Error ? error.message : 'Unknown error', error);
|
|
81
170
|
}
|
|
@@ -83,10 +172,9 @@ export async function handleModelRegistryList() {
|
|
|
83
172
|
export async function handleModelRegistrySearch(request) {
|
|
84
173
|
logger.debug('Handling QVAC model registry search request', request);
|
|
85
174
|
try {
|
|
86
|
-
const
|
|
87
|
-
const registryModels = await client.findBy({
|
|
175
|
+
const registryModels = await withRegistryClient(request, (client) => client.findBy({
|
|
88
176
|
...(request.quantization && { quantization: request.quantization })
|
|
89
|
-
});
|
|
177
|
+
}));
|
|
90
178
|
let models = registryModels
|
|
91
179
|
.map(processRegistryModel)
|
|
92
180
|
.filter((m) => m !== null);
|
|
@@ -113,23 +201,17 @@ export async function handleModelRegistrySearch(request) {
|
|
|
113
201
|
}
|
|
114
202
|
catch (error) {
|
|
115
203
|
logger.error('QVAC model registry search failed:', error);
|
|
116
|
-
|
|
117
|
-
const qvacError = error;
|
|
118
|
-
if (qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CONNECT ||
|
|
119
|
-
qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CLOSE ||
|
|
120
|
-
qvacError.code === REGISTRY_ERROR_CODES.MODEL_NOT_FOUND) {
|
|
204
|
+
if (isPassThrough(error))
|
|
121
205
|
throw error;
|
|
122
|
-
}
|
|
123
206
|
throw new ModelRegistryQueryFailedError(error instanceof Error ? error.message : 'Unknown error', error);
|
|
124
207
|
}
|
|
125
208
|
}
|
|
126
209
|
export async function handleModelRegistryGetModel(request) {
|
|
127
210
|
logger.debug('Handling QVAC model registry get model request', request);
|
|
128
211
|
try {
|
|
129
|
-
const
|
|
130
|
-
const rawModel = await client.getModel(request.registryPath, request.registrySource);
|
|
212
|
+
const rawModel = await withRegistryClient(request, (client) => client.getModel(request.registryPath, request.registrySource));
|
|
131
213
|
if (!rawModel) {
|
|
132
|
-
throw new
|
|
214
|
+
throw new ModelRegistryModelNotFoundError(request.registrySource, request.registryPath);
|
|
133
215
|
}
|
|
134
216
|
const model = processRegistryModel(rawModel);
|
|
135
217
|
if (!model) {
|
|
@@ -143,14 +225,11 @@ export async function handleModelRegistryGetModel(request) {
|
|
|
143
225
|
};
|
|
144
226
|
}
|
|
145
227
|
catch (error) {
|
|
228
|
+
if (error instanceof ModelRegistryModelNotFoundError)
|
|
229
|
+
throw error;
|
|
146
230
|
logger.error('QVAC model registry get model failed:', error);
|
|
147
|
-
|
|
148
|
-
const qvacError = error;
|
|
149
|
-
if (qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CONNECT ||
|
|
150
|
-
qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CLOSE ||
|
|
151
|
-
qvacError.code === REGISTRY_ERROR_CODES.MODEL_NOT_FOUND) {
|
|
231
|
+
if (isPassThrough(error))
|
|
152
232
|
throw error;
|
|
153
|
-
}
|
|
154
233
|
throw new ModelRegistryQueryFailedError(error instanceof Error ? error.message : 'Unknown error', error);
|
|
155
234
|
}
|
|
156
235
|
}
|
|
@@ -73,11 +73,11 @@ async function run(operation, work) {
|
|
|
73
73
|
throw new RpcServerOperationError(operation, error instanceof Error ? error.message : String(error), error);
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
|
-
async function withContext(kind, requestId, work) {
|
|
76
|
+
async function withContext(kind, requestId, sessionId, work) {
|
|
77
77
|
const task = (async () => {
|
|
78
78
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
79
79
|
try {
|
|
80
|
-
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({ requestId, kind }), true);
|
|
80
|
+
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({ requestId, sessionId, kind }), true);
|
|
81
81
|
return await work(ctx);
|
|
82
82
|
}
|
|
83
83
|
catch (e_1) {
|
|
@@ -90,7 +90,7 @@ async function withContext(kind, requestId, work) {
|
|
|
90
90
|
await result_1;
|
|
91
91
|
}
|
|
92
92
|
})();
|
|
93
|
-
requests.set(task, requestId);
|
|
93
|
+
requests.set(task, { requestId, sessionId });
|
|
94
94
|
try {
|
|
95
95
|
return await task;
|
|
96
96
|
}
|
|
@@ -105,7 +105,7 @@ async function withContext(kind, requestId, work) {
|
|
|
105
105
|
}
|
|
106
106
|
}
|
|
107
107
|
export async function handleStartRpcServer(request) {
|
|
108
|
-
return run('startRpcServer', () => withContext('rpcServer', request.requestId ?? generateRandomRequestId(), async (ctx) => ({
|
|
108
|
+
return run('startRpcServer', () => withContext('rpcServer', request.requestId ?? generateRandomRequestId(), request.sessionId, async (ctx) => ({
|
|
109
109
|
type: 'startRpcServer',
|
|
110
110
|
...(await rpcServers.start(request, ctx))
|
|
111
111
|
})));
|
|
@@ -117,7 +117,7 @@ export async function handleStopRpcServer(request) {
|
|
|
117
117
|
});
|
|
118
118
|
}
|
|
119
119
|
export async function handleDiscoverRpcServers(request) {
|
|
120
|
-
return run('discoverRpcServers', () => withContext('rpcDiscovery', request.requestId ?? generateRandomRequestId(), async (ctx) => ({
|
|
120
|
+
return run('discoverRpcServers', () => withContext('rpcDiscovery', request.requestId ?? generateRandomRequestId(), request.sessionId, async (ctx) => ({
|
|
121
121
|
type: 'discoverRpcServers',
|
|
122
122
|
servers: await discoverRpcEndpoints(request.topic, request.timeoutMs ?? 5000, ctx)
|
|
123
123
|
})));
|
|
@@ -126,8 +126,8 @@ export function closeRpcResources() {
|
|
|
126
126
|
if (closing)
|
|
127
127
|
return closing;
|
|
128
128
|
closing = (async () => {
|
|
129
|
-
for (const
|
|
130
|
-
getRequestRegistry().cancel(
|
|
129
|
+
for (const target of requests.values())
|
|
130
|
+
getRequestRegistry().cancel(target);
|
|
131
131
|
const stopping = rpcServers.close();
|
|
132
132
|
const results = await Promise.allSettled([stopping, ...requests.keys()]);
|
|
133
133
|
const stopResult = results[0];
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { OpenSessionRequest, OpenSessionResponse, CloseSessionRequest, CloseSessionResponse } from '../schemas/session.js';
|
|
2
|
+
export declare function handleOpenSession(request: OpenSessionRequest): OpenSessionResponse;
|
|
3
|
+
/**
|
|
4
|
+
* Closing a session cancels its requests, waits for them to leave the
|
|
5
|
+
* registry, then disposes the vector indexes it opened and releases every
|
|
6
|
+
* model lease it holds. A model whose last lease was released is unloaded.
|
|
7
|
+
* The session record is removed first so a
|
|
8
|
+
* call racing the close is refused at dispatch instead of starting work on a
|
|
9
|
+
* session that is going away. Its id is not reusable until the close returns.
|
|
10
|
+
*/
|
|
11
|
+
export declare function handleCloseSession(request: CloseSessionRequest): Promise<CloseSessionResponse>;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { openSession, closeSession, finishSessionClose } from '../runtime/session-registry.js';
|
|
2
|
+
import { getRequestRegistry } from '../runtime/request-context.js';
|
|
3
|
+
import { getModelIdsLeasedBy } from '../runtime/model-registry.js';
|
|
4
|
+
import { disposeSessionVectorIndexes } from '../runtime/vector-index-registry.js';
|
|
5
|
+
import { unloadModel } from '../plugins/ops/unload-model.js';
|
|
6
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
7
|
+
const logger = getEngineLogger();
|
|
8
|
+
export function handleOpenSession(request) {
|
|
9
|
+
const entry = openSession(request.sessionId, request.name);
|
|
10
|
+
return { type: 'openSession', success: true, sessionId: entry.id };
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Closing a session cancels its requests, waits for them to leave the
|
|
14
|
+
* registry, then disposes the vector indexes it opened and releases every
|
|
15
|
+
* model lease it holds. A model whose last lease was released is unloaded.
|
|
16
|
+
* The session record is removed first so a
|
|
17
|
+
* call racing the close is refused at dispatch instead of starting work on a
|
|
18
|
+
* session that is going away. Its id is not reusable until the close returns.
|
|
19
|
+
*/
|
|
20
|
+
export async function handleCloseSession(request) {
|
|
21
|
+
const { sessionId } = request;
|
|
22
|
+
closeSession(sessionId);
|
|
23
|
+
try {
|
|
24
|
+
const cancelled = await getRequestRegistry().drainSession(sessionId, 'sessionClosed');
|
|
25
|
+
disposeSessionVectorIndexes(sessionId);
|
|
26
|
+
const releasedModels = getModelIdsLeasedBy(sessionId);
|
|
27
|
+
for (const modelId of releasedModels) {
|
|
28
|
+
try {
|
|
29
|
+
await unloadModel({ modelId, sessionId });
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
logger.error(`Error releasing model ${modelId} for session ${sessionId}:`, error);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return { type: 'closeSession', success: true, sessionId, cancelled, releasedModels };
|
|
36
|
+
}
|
|
37
|
+
finally {
|
|
38
|
+
finishSessionClose(sessionId);
|
|
39
|
+
}
|
|
40
|
+
}
|
package/dist/handlers/types.d.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import type { Request, Response } from '../schemas/index.js';
|
|
1
|
+
import type { AbortSignalLike, Request, Response } from '../schemas/index.js';
|
|
2
|
+
/** Passed to a stream handler with `endsOnAbort`. */
|
|
3
|
+
export interface StreamHandlerContext {
|
|
4
|
+
/** Aborts when the caller stops reading. */
|
|
5
|
+
signal?: AbortSignalLike;
|
|
6
|
+
}
|
|
2
7
|
export type ReplyHandler = (request: any, ...args: any[]) => Promise<Response> | Response;
|
|
3
8
|
export type StreamHandler = (request: any, ...args: any[]) => AsyncGenerator<Response>;
|
|
4
9
|
export type ProgressHandler = (request: any, ...args: any[]) => Promise<Response>;
|
|
@@ -8,4 +13,5 @@ export type HandlerEntry = {
|
|
|
8
13
|
handler: ReplyHandler | StreamHandler | ProgressHandler | DuplexStreamHandler;
|
|
9
14
|
supportsProgress?: boolean | ((request: Request) => boolean);
|
|
10
15
|
pluginOp?: boolean;
|
|
16
|
+
endsOnAbort?: boolean;
|
|
11
17
|
};
|