@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { AbortSignalLike } from '../schemas/common.js';
|
|
2
2
|
import type { RequestContext, RequestKind } from '../runtime/request-context.js';
|
|
3
3
|
import type { Logger } from '../logging/types.js';
|
|
4
4
|
/**
|
|
@@ -10,6 +10,12 @@ export type RequestOutcome = 'completed' | 'failed' | 'cancelled';
|
|
|
10
10
|
export interface BeginOpts {
|
|
11
11
|
/** Stable identity. Caller-provided so a `cancel` can target it by id. */
|
|
12
12
|
requestId: string;
|
|
13
|
+
/**
|
|
14
|
+
* Caller session the request belongs to. The registry keys every request by
|
|
15
|
+
* `(sessionId, requestId)`, so a cancel from another session never finds it.
|
|
16
|
+
* Defaults to the owner session.
|
|
17
|
+
*/
|
|
18
|
+
sessionId?: string | undefined;
|
|
13
19
|
kind: RequestKind;
|
|
14
20
|
modelId?: string;
|
|
15
21
|
/**
|
|
@@ -18,7 +24,9 @@ export interface BeginOpts {
|
|
|
18
24
|
* Composes through a `addEventListener("abort", ...)` hook so cancelling
|
|
19
25
|
* the parent does not require iterating the registry.
|
|
20
26
|
*/
|
|
21
|
-
parentSignal?:
|
|
27
|
+
parentSignal?: AbortSignalLike & {
|
|
28
|
+
readonly reason?: unknown;
|
|
29
|
+
};
|
|
22
30
|
/**
|
|
23
31
|
* Per-request override of the kind policy's `maxConcurrentPerModel`. The
|
|
24
32
|
* policy value is one number for every model of a kind, but handlers may
|
|
@@ -34,14 +42,26 @@ export interface BeginOpts {
|
|
|
34
42
|
}
|
|
35
43
|
export interface CancelByRequestId {
|
|
36
44
|
requestId: string;
|
|
45
|
+
/** Session the request belongs to. Defaults to the owner session. */
|
|
46
|
+
sessionId?: string | undefined;
|
|
37
47
|
reason?: string;
|
|
38
48
|
}
|
|
39
49
|
export interface CancelByModelId {
|
|
40
50
|
modelId: string;
|
|
41
51
|
kind?: RequestKind;
|
|
52
|
+
/** Only this session's requests on the model. Defaults to the owner session. */
|
|
53
|
+
sessionId?: string | undefined;
|
|
54
|
+
/** Every session's requests on the model. Set by the owner and by teardown. */
|
|
55
|
+
allSessions?: boolean;
|
|
56
|
+
reason?: string;
|
|
57
|
+
}
|
|
58
|
+
/** Every request of one session, on every model: the `closeSession` sweep. */
|
|
59
|
+
export interface CancelBySession {
|
|
60
|
+
sessionId: string;
|
|
61
|
+
allModels: true;
|
|
42
62
|
reason?: string;
|
|
43
63
|
}
|
|
44
|
-
export type CancelTarget = CancelByRequestId | CancelByModelId;
|
|
64
|
+
export type CancelTarget = CancelByRequestId | CancelByModelId | CancelBySession;
|
|
45
65
|
/**
|
|
46
66
|
* Per-kind admission rule. Kinds without a registered policy have no
|
|
47
67
|
* admission control (every `begin(...)` is accepted as long as the
|
|
@@ -162,8 +182,8 @@ export interface RequestRegistry {
|
|
|
162
182
|
* // → rejects with RequestRejectedByPolicyError (code 52420)
|
|
163
183
|
*/
|
|
164
184
|
policy(opts: ConcurrencyPolicy): void;
|
|
165
|
-
/** Look up an in-flight request by id. */
|
|
166
|
-
get(requestId: string): RequestContext | null;
|
|
185
|
+
/** Look up an in-flight request by id within a session (owner by default). */
|
|
186
|
+
get(requestId: string, sessionId?: string): RequestContext | null;
|
|
167
187
|
/**
|
|
168
188
|
* Snapshot of currently-tracked requests. Useful for diagnostics /
|
|
169
189
|
* structured logs ("which requests are in flight right now?"). Returns
|
|
@@ -177,7 +197,11 @@ export interface RequestRegistry {
|
|
|
177
197
|
*
|
|
178
198
|
* For `{ modelId }` and an optional `kind`, cancels every active
|
|
179
199
|
* request that matches the predicate. This is the broad-cancel path
|
|
180
|
-
* the pre-registry `cancel({ modelId })` API maps to.
|
|
200
|
+
* the pre-registry `cancel({ modelId })` API maps to. Without
|
|
201
|
+
* `allSessions` it only reaches the target session's requests.
|
|
202
|
+
*
|
|
203
|
+
* For `{ sessionId, allModels: true }`, cancels every request the session
|
|
204
|
+
* owns, active or queued.
|
|
181
205
|
*/
|
|
182
206
|
cancel(target: CancelTarget): number;
|
|
183
207
|
/**
|
|
@@ -196,16 +220,40 @@ export interface RequestRegistry {
|
|
|
196
220
|
* mid-teardown, and no new request goes active against it during teardown.
|
|
197
221
|
*/
|
|
198
222
|
withModelDraining(modelId: string, teardown: () => Promise<void>): Promise<void>;
|
|
223
|
+
/**
|
|
224
|
+
* Cancel every request of a session, active or queued, and wait for the
|
|
225
|
+
* active ones to dispose. Returns the number of requests this call cancelled.
|
|
226
|
+
*/
|
|
227
|
+
drainSession(sessionId: string, reason: string): Promise<number>;
|
|
199
228
|
/**
|
|
200
229
|
* Mark a request finished and dispose its scope. Equivalent to
|
|
201
230
|
* `await ctx[Symbol.asyncDispose]()` with an explicit outcome.
|
|
202
231
|
* Idempotent — calling `end` after a scope dispose is a no-op.
|
|
203
232
|
*/
|
|
204
|
-
end(requestId: string, outcome: RequestOutcome): Promise<void>;
|
|
233
|
+
end(requestId: string, outcome: RequestOutcome, sessionId?: string): Promise<void>;
|
|
234
|
+
/**
|
|
235
|
+
* Make `signal` the parent signal of the request the caller is about to
|
|
236
|
+
* begin under `(sessionId, requestId)`, unless that begin passes its own
|
|
237
|
+
* `parentSignal`. The host binds the transport's signal of a call before
|
|
238
|
+
* dispatching it, so a cancel over the transport aborts the call's work. A
|
|
239
|
+
* call whose id is already in use binds nothing: its begin is refused, and
|
|
240
|
+
* its signal must not reach the request that holds the id. Returns a
|
|
241
|
+
* function that removes the binding; call it once the call has finished.
|
|
242
|
+
*/
|
|
243
|
+
bindTransportSignal(requestId: string, sessionId: string | undefined, signal: AbortSignalLike & {
|
|
244
|
+
readonly reason?: unknown;
|
|
245
|
+
}): () => void;
|
|
205
246
|
}
|
|
206
247
|
export declare function createRequestRegistry(options?: {
|
|
207
248
|
/** Defaults to `getEngineLogger()`. Tests inject a stub. */
|
|
208
249
|
logger?: Logger;
|
|
250
|
+
/**
|
|
251
|
+
* Throws when a request may not begin under `sessionId`, called for every
|
|
252
|
+
* session other than the owner. Dispatch checks the session too, but a
|
|
253
|
+
* handler that awaits before its `begin` can reach it after the session
|
|
254
|
+
* closed; this refuses that late begin.
|
|
255
|
+
*/
|
|
256
|
+
assertSessionOpen?: (sessionId: string) => void;
|
|
209
257
|
}): RequestRegistry;
|
|
210
258
|
/**
|
|
211
259
|
* Test-only knobs exported for `request-registry.test.ts` so the bound
|
|
@@ -2,6 +2,7 @@ import { AbortController } from 'bare-abort-controller';
|
|
|
2
2
|
import { createDisposableScope } from '../runtime/disposable-scope.js';
|
|
3
3
|
import { RequestIdConflictError, RequestRejectedByPolicyError } from '../errors/index.js';
|
|
4
4
|
import { getEngineLogger } from '../logging/index.js';
|
|
5
|
+
import { OWNER_SESSION_ID, requestRegistryKey } from '../runtime/session-id.js';
|
|
5
6
|
/**
|
|
6
7
|
* Tuning knobs for the "cancelled-before-begin" bookkeeping set.
|
|
7
8
|
*
|
|
@@ -33,6 +34,9 @@ const CANCEL_BEFORE_BEGIN_TTL_MS = 30_000;
|
|
|
33
34
|
*/
|
|
34
35
|
const DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL = 64;
|
|
35
36
|
export function createRequestRegistry(options) {
|
|
37
|
+
// Every per-request map below is keyed by `requestRegistryKey(sessionId,
|
|
38
|
+
// requestId)`, so two sessions reusing one request id never collide and a
|
|
39
|
+
// cancel from one session cannot reach the other's request.
|
|
36
40
|
const entries = new Map();
|
|
37
41
|
const policies = new Map();
|
|
38
42
|
const logger = options?.logger ?? getEngineLogger();
|
|
@@ -49,12 +53,19 @@ export function createRequestRegistry(options) {
|
|
|
49
53
|
*/
|
|
50
54
|
const waitersById = new Map();
|
|
51
55
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
+
* Begins that have started but not yet landed in `entries`: reserved
|
|
57
|
+
* synchronously at `begin(...)` start and cleared once the entry lands (or
|
|
58
|
+
* the begin throws). A same-tick second `begin(...)` with the same id runs
|
|
59
|
+
* before the first reaches `entries`/`waitersById`, so without this it
|
|
60
|
+
* would slip past the duplicate check and both would execute. A session
|
|
61
|
+
* drain also consults it: a begin suspended in admission is in neither
|
|
62
|
+
* `entries` nor (once granted) `waitersById`, and would otherwise start its
|
|
63
|
+
* work after the session closed.
|
|
56
64
|
*/
|
|
57
|
-
const
|
|
65
|
+
const pendingBegins = new Map();
|
|
66
|
+
// The transport signal of each call the host is dispatching, by the key its
|
|
67
|
+
// request begins under: see `bindTransportSignal`.
|
|
68
|
+
const transportSignals = new Map();
|
|
58
69
|
// Models being torn down: a begin(...) for one starts aborted. The barrier
|
|
59
70
|
// half of cancelAndDrain — catches begins its scan can't see yet.
|
|
60
71
|
const drainingModels = new Set();
|
|
@@ -70,7 +81,7 @@ export function createRequestRegistry(options) {
|
|
|
70
81
|
const disposingIds = new Set();
|
|
71
82
|
function logLifecycle(event, ctx, durationMs) {
|
|
72
83
|
const modelId = ctx.modelId !== undefined ? ctx.modelId : '-';
|
|
73
|
-
const base = `[request-lifecycle] ${event} requestId=${ctx.requestId} kind=${ctx.kind} modelId=${modelId} state=${ctx.state}`;
|
|
84
|
+
const base = `[request-lifecycle] ${event} sessionId=${ctx.sessionId} requestId=${ctx.requestId} kind=${ctx.kind} modelId=${modelId} state=${ctx.state}`;
|
|
74
85
|
const line = durationMs !== undefined ? `${base} durationMs=${durationMs}` : base;
|
|
75
86
|
// `failed` end emits at `warn` so log shippers can alert on
|
|
76
87
|
// `level>=warn` for this prefix without parsing `state=failed`
|
|
@@ -113,15 +124,15 @@ export function createRequestRegistry(options) {
|
|
|
113
124
|
cancelledBeforeBegin.delete(id);
|
|
114
125
|
}
|
|
115
126
|
}
|
|
116
|
-
function recordCancelBeforeBegin(
|
|
127
|
+
function recordCancelBeforeBegin(key, reason) {
|
|
117
128
|
const now = Date.now();
|
|
118
129
|
pruneCancelBeforeBeginExpired(now);
|
|
119
130
|
// Re-canceling an id that is already tracked refreshes its TTL but
|
|
120
131
|
// keeps the original reason — the first cancel "won" the race.
|
|
121
|
-
if (cancelledBeforeBegin.has(
|
|
122
|
-
const existing = cancelledBeforeBegin.get(
|
|
123
|
-
cancelledBeforeBegin.delete(
|
|
124
|
-
cancelledBeforeBegin.set(
|
|
132
|
+
if (cancelledBeforeBegin.has(key)) {
|
|
133
|
+
const existing = cancelledBeforeBegin.get(key);
|
|
134
|
+
cancelledBeforeBegin.delete(key);
|
|
135
|
+
cancelledBeforeBegin.set(key, { ...existing, at: now });
|
|
125
136
|
return;
|
|
126
137
|
}
|
|
127
138
|
if (cancelledBeforeBegin.size >= CANCEL_BEFORE_BEGIN_MAX_ENTRIES) {
|
|
@@ -129,14 +140,14 @@ export function createRequestRegistry(options) {
|
|
|
129
140
|
if (oldest !== undefined)
|
|
130
141
|
cancelledBeforeBegin.delete(oldest);
|
|
131
142
|
}
|
|
132
|
-
cancelledBeforeBegin.set(
|
|
143
|
+
cancelledBeforeBegin.set(key, reason !== undefined ? { at: now, reason } : { at: now });
|
|
133
144
|
}
|
|
134
|
-
function consumeCancelBeforeBegin(
|
|
145
|
+
function consumeCancelBeforeBegin(key) {
|
|
135
146
|
pruneCancelBeforeBeginExpired();
|
|
136
|
-
const entry = cancelledBeforeBegin.get(
|
|
147
|
+
const entry = cancelledBeforeBegin.get(key);
|
|
137
148
|
if (!entry)
|
|
138
149
|
return undefined;
|
|
139
|
-
cancelledBeforeBegin.delete(
|
|
150
|
+
cancelledBeforeBegin.delete(key);
|
|
140
151
|
return entry;
|
|
141
152
|
}
|
|
142
153
|
function slotKey(lane, modelId) {
|
|
@@ -157,7 +168,7 @@ export function createRequestRegistry(options) {
|
|
|
157
168
|
clearTimeout(waiter.timer);
|
|
158
169
|
waiter.timer = undefined;
|
|
159
170
|
}
|
|
160
|
-
waitersById.delete(waiter.
|
|
171
|
+
waitersById.delete(waiter.key);
|
|
161
172
|
const st = keyStates.get(key);
|
|
162
173
|
if (!st)
|
|
163
174
|
return;
|
|
@@ -187,7 +198,7 @@ export function createRequestRegistry(options) {
|
|
|
187
198
|
* across a hand-off — that's what guarantees FIFO fairness with no
|
|
188
199
|
* acquire/release race in the single-threaded event loop.
|
|
189
200
|
*/
|
|
190
|
-
async function acquireSlot(opts) {
|
|
201
|
+
async function acquireSlot(opts, sessionId, registryKey) {
|
|
191
202
|
if (opts.modelId === undefined)
|
|
192
203
|
return { slotKey: undefined, exclusive: false };
|
|
193
204
|
const policy = policies.get(opts.kind);
|
|
@@ -240,29 +251,50 @@ export function createRequestRegistry(options) {
|
|
|
240
251
|
if (st.waiters.length >= policy.maxQueueDepth) {
|
|
241
252
|
throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `request queue for this model is full (${st.waiters.length} waiting)`);
|
|
242
253
|
}
|
|
254
|
+
// A parent abort while queued must not leave this begin behind a busy lane
|
|
255
|
+
// until the lane frees: pull it out the way a cancel by id would, so it
|
|
256
|
+
// resumes at once into an aborted context. A parent abort after the grant
|
|
257
|
+
// is seen when the context is created.
|
|
258
|
+
const parent = opts.parentSignal;
|
|
259
|
+
const onParentAbortWhileQueued = () => {
|
|
260
|
+
const queued = waitersById.get(registryKey);
|
|
261
|
+
if (!queued)
|
|
262
|
+
return;
|
|
263
|
+
const reason = typeof parent?.reason === 'string' ? parent.reason : undefined;
|
|
264
|
+
cancelQueuedWaiterGracefully(queued.key, queued.waiter, reason);
|
|
265
|
+
};
|
|
266
|
+
parent?.addEventListener('abort', onParentAbortWhileQueued, { once: true });
|
|
243
267
|
const queue = st;
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
268
|
+
let granted;
|
|
269
|
+
try {
|
|
270
|
+
granted = await new Promise((resolve, reject) => {
|
|
271
|
+
const waiter = {
|
|
272
|
+
requestId: opts.requestId,
|
|
273
|
+
sessionId,
|
|
274
|
+
key: registryKey,
|
|
275
|
+
kind: opts.kind,
|
|
276
|
+
modelId,
|
|
277
|
+
enqueuedAt: Date.now(),
|
|
278
|
+
resolve,
|
|
279
|
+
reject,
|
|
280
|
+
timer: undefined,
|
|
281
|
+
exclusive,
|
|
282
|
+
cap: maxConcurrent
|
|
283
|
+
};
|
|
284
|
+
if (policy.queueTimeoutMs !== undefined) {
|
|
285
|
+
const timeoutMs = policy.queueTimeoutMs;
|
|
286
|
+
waiter.timer = setTimeout(() => {
|
|
287
|
+
removeWaiter(key, waiter);
|
|
288
|
+
reject(new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `timed out waiting ${timeoutMs}ms for a slot on this model`));
|
|
289
|
+
}, timeoutMs);
|
|
290
|
+
}
|
|
291
|
+
queue.waiters.push(waiter);
|
|
292
|
+
waitersById.set(registryKey, { key, waiter });
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
finally {
|
|
296
|
+
parent?.removeEventListener('abort', onParentAbortWhileQueued);
|
|
297
|
+
}
|
|
266
298
|
return granted ? { slotKey: key, exclusive } : { slotKey: undefined, exclusive: false };
|
|
267
299
|
}
|
|
268
300
|
/**
|
|
@@ -290,7 +322,7 @@ export function createRequestRegistry(options) {
|
|
|
290
322
|
st.readerCap = head.cap;
|
|
291
323
|
}
|
|
292
324
|
st.waiters.shift();
|
|
293
|
-
waitersById.delete(head.
|
|
325
|
+
waitersById.delete(head.key);
|
|
294
326
|
if (head.timer !== undefined) {
|
|
295
327
|
clearTimeout(head.timer);
|
|
296
328
|
head.timer = undefined;
|
|
@@ -332,13 +364,13 @@ export function createRequestRegistry(options) {
|
|
|
332
364
|
disposingEntries.add(entry);
|
|
333
365
|
// Keep the id reserved across async teardown so a same-id begin can't
|
|
334
366
|
// activate concurrently and a cancel can't strand a tripwire for a reuse.
|
|
335
|
-
disposingIds.add(entry.ctx.
|
|
367
|
+
disposingIds.add(entry.ctx.registryKey);
|
|
336
368
|
entry.ctx.state = outcome;
|
|
337
369
|
entry.detachParent();
|
|
338
370
|
logLifecycle('end', entry.ctx, Date.now() - entry.startedAt);
|
|
339
371
|
// Pull the entry out before unwinding so observers (e.g. a `cancel(...)`
|
|
340
372
|
// racing with dispose) don't see a half-disposed context.
|
|
341
|
-
entries.delete(entry.ctx.
|
|
373
|
+
entries.delete(entry.ctx.registryKey);
|
|
342
374
|
try {
|
|
343
375
|
await entry.scope[Symbol.asyncDispose]();
|
|
344
376
|
}
|
|
@@ -353,13 +385,37 @@ export function createRequestRegistry(options) {
|
|
|
353
385
|
releaseSlot(entry.slotKey, entry.exclusive);
|
|
354
386
|
disposingEntries.delete(entry);
|
|
355
387
|
// The id is free for reuse only now that native teardown is complete.
|
|
356
|
-
disposingIds.delete(entry.ctx.
|
|
388
|
+
disposingIds.delete(entry.ctx.registryKey);
|
|
357
389
|
// Signal any `cancelAndDrain` waiting on this request that its native
|
|
358
390
|
// teardown is complete.
|
|
359
391
|
entry.resolveDisposed();
|
|
360
392
|
}
|
|
361
393
|
}
|
|
362
|
-
|
|
394
|
+
function idInUse(registryKey) {
|
|
395
|
+
return (entries.has(registryKey) ||
|
|
396
|
+
waitersById.has(registryKey) ||
|
|
397
|
+
pendingBegins.has(registryKey) ||
|
|
398
|
+
disposingIds.has(registryKey));
|
|
399
|
+
}
|
|
400
|
+
function bindTransportSignal(requestId, sessionId, signal) {
|
|
401
|
+
const registryKey = requestRegistryKey(sessionId ?? OWNER_SESSION_ID, requestId);
|
|
402
|
+
if (transportSignals.has(registryKey) || idInUse(registryKey))
|
|
403
|
+
return () => { };
|
|
404
|
+
transportSignals.set(registryKey, signal);
|
|
405
|
+
return () => {
|
|
406
|
+
if (transportSignals.get(registryKey) === signal)
|
|
407
|
+
transportSignals.delete(registryKey);
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
async function begin(beginOpts) {
|
|
411
|
+
const sessionId = beginOpts.sessionId ?? OWNER_SESSION_ID;
|
|
412
|
+
if (sessionId !== OWNER_SESSION_ID)
|
|
413
|
+
options?.assertSessionOpen?.(sessionId);
|
|
414
|
+
const registryKey = requestRegistryKey(sessionId, beginOpts.requestId);
|
|
415
|
+
const transportSignal = transportSignals.get(registryKey);
|
|
416
|
+
const opts = beginOpts.parentSignal || !transportSignal
|
|
417
|
+
? beginOpts
|
|
418
|
+
: { ...beginOpts, parentSignal: transportSignal };
|
|
363
419
|
// A request id is reserved for its whole lifecycle: the time it spends
|
|
364
420
|
// *queued* for an admission slot, while it is active, and while it is
|
|
365
421
|
// tearing down. `waitersById` holds the still-queued begins (not yet in
|
|
@@ -368,19 +424,24 @@ export function createRequestRegistry(options) {
|
|
|
368
424
|
// otherwise a second begin with the same id would enqueue behind the first
|
|
369
425
|
// and overwrite its `waitersById` index (unreachable by `cancel(...)`), or
|
|
370
426
|
// activate while the prior request is still releasing its native context.
|
|
371
|
-
if (
|
|
372
|
-
waitersById.has(opts.requestId) ||
|
|
373
|
-
reservedIds.has(opts.requestId) ||
|
|
374
|
-
disposingIds.has(opts.requestId)) {
|
|
427
|
+
if (idInUse(registryKey)) {
|
|
375
428
|
throw new RequestIdConflictError(opts.requestId);
|
|
376
429
|
}
|
|
377
430
|
// Reserve synchronously so a same-tick duplicate begin is rejected before
|
|
378
|
-
// this one reaches `entries`/`waitersById`.
|
|
379
|
-
|
|
431
|
+
// this one reaches `entries`/`waitersById`. Settled on every exit below.
|
|
432
|
+
let settle = () => { };
|
|
433
|
+
const settled = new Promise((resolve) => {
|
|
434
|
+
settle = resolve;
|
|
435
|
+
});
|
|
436
|
+
pendingBegins.set(registryKey, { sessionId, settled, settle });
|
|
437
|
+
const settlePending = () => {
|
|
438
|
+
pendingBegins.delete(registryKey);
|
|
439
|
+
settle();
|
|
440
|
+
};
|
|
380
441
|
// A request cancelled BEFORE begin must not queue for a slot on a saturated
|
|
381
442
|
// lane: consume its marker first and skip admission, so it resolves at once
|
|
382
443
|
// into an aborted context holding no slot instead of waiting for the holder.
|
|
383
|
-
let preCancel = consumeCancelBeforeBegin(
|
|
444
|
+
let preCancel = consumeCancelBeforeBegin(registryKey);
|
|
384
445
|
// Admission control runs before allocation so a rejected begin leaves no
|
|
385
446
|
// controller / scope behind. This may await: a gated `(kind, modelId)` at
|
|
386
447
|
// capacity queues the begin FIFO and resolves once a slot frees (or rejects
|
|
@@ -392,10 +453,10 @@ export function createRequestRegistry(options) {
|
|
|
392
453
|
// and observers still see the request registered on the same tick.
|
|
393
454
|
let acquired;
|
|
394
455
|
try {
|
|
395
|
-
acquired = await acquireSlot(opts);
|
|
456
|
+
acquired = await acquireSlot(opts, sessionId, registryKey);
|
|
396
457
|
}
|
|
397
458
|
catch (err) {
|
|
398
|
-
|
|
459
|
+
settlePending();
|
|
399
460
|
throw err;
|
|
400
461
|
}
|
|
401
462
|
slotKey = acquired.slotKey;
|
|
@@ -404,15 +465,15 @@ export function createRequestRegistry(options) {
|
|
|
404
465
|
// await above. A duplicate id could (astronomically unlikely) have landed
|
|
405
466
|
// while we were queued — re-check and hand the just-acquired slot back
|
|
406
467
|
// rather than stranding it on the conflicting throw.
|
|
407
|
-
if (entries.has(
|
|
468
|
+
if (entries.has(registryKey)) {
|
|
408
469
|
if (slotKey !== undefined)
|
|
409
470
|
releaseSlot(slotKey, exclusive);
|
|
410
|
-
|
|
471
|
+
settlePending();
|
|
411
472
|
throw new RequestIdConflictError(opts.requestId);
|
|
412
473
|
}
|
|
413
474
|
// A cancel that landed WHILE we were queued resolved acquireSlot with no
|
|
414
475
|
// slot and recorded a marker; consume it so the context starts aborted.
|
|
415
|
-
preCancel = consumeCancelBeforeBegin(
|
|
476
|
+
preCancel = consumeCancelBeforeBegin(registryKey);
|
|
416
477
|
}
|
|
417
478
|
// A begin arriving while its model is being torn down starts aborted — no
|
|
418
479
|
// native work against a model about to be freed.
|
|
@@ -439,6 +500,8 @@ export function createRequestRegistry(options) {
|
|
|
439
500
|
}
|
|
440
501
|
const ctx = {
|
|
441
502
|
requestId: opts.requestId,
|
|
503
|
+
sessionId,
|
|
504
|
+
registryKey,
|
|
442
505
|
kind: opts.kind,
|
|
443
506
|
modelId: opts.modelId,
|
|
444
507
|
signal: controller.signal,
|
|
@@ -467,13 +530,19 @@ export function createRequestRegistry(options) {
|
|
|
467
530
|
disposed,
|
|
468
531
|
resolveDisposed
|
|
469
532
|
};
|
|
470
|
-
entries.set(
|
|
471
|
-
|
|
533
|
+
entries.set(registryKey, entry);
|
|
534
|
+
settlePending();
|
|
472
535
|
logLifecycle('begin', ctx);
|
|
473
536
|
return {
|
|
474
537
|
get requestId() {
|
|
475
538
|
return ctx.requestId;
|
|
476
539
|
},
|
|
540
|
+
get sessionId() {
|
|
541
|
+
return ctx.sessionId;
|
|
542
|
+
},
|
|
543
|
+
get registryKey() {
|
|
544
|
+
return ctx.registryKey;
|
|
545
|
+
},
|
|
477
546
|
get kind() {
|
|
478
547
|
return ctx.kind;
|
|
479
548
|
},
|
|
@@ -497,8 +566,8 @@ export function createRequestRegistry(options) {
|
|
|
497
566
|
}
|
|
498
567
|
};
|
|
499
568
|
}
|
|
500
|
-
function get(requestId) {
|
|
501
|
-
return entries.get(requestId)?.ctx ?? null;
|
|
569
|
+
function get(requestId, sessionId = OWNER_SESSION_ID) {
|
|
570
|
+
return entries.get(requestRegistryKey(sessionId, requestId))?.ctx ?? null;
|
|
502
571
|
}
|
|
503
572
|
function list() {
|
|
504
573
|
return Array.from(entries.values(), (e) => e.ctx);
|
|
@@ -511,14 +580,15 @@ export function createRequestRegistry(options) {
|
|
|
511
580
|
* a clean `cancelled` state rather than throwing.
|
|
512
581
|
*/
|
|
513
582
|
function cancelQueuedWaiterGracefully(key, waiter, reason) {
|
|
514
|
-
recordCancelBeforeBegin(waiter.
|
|
583
|
+
recordCancelBeforeBegin(waiter.key, reason);
|
|
515
584
|
removeWaiter(key, waiter);
|
|
516
585
|
waiter.resolve(false);
|
|
517
586
|
}
|
|
518
587
|
function cancel(target) {
|
|
519
588
|
let cancelled = 0;
|
|
520
589
|
if ('requestId' in target) {
|
|
521
|
-
const
|
|
590
|
+
const registryKey = requestRegistryKey(target.sessionId ?? OWNER_SESSION_ID, target.requestId);
|
|
591
|
+
const entry = entries.get(registryKey);
|
|
522
592
|
if (entry) {
|
|
523
593
|
if (cancelEntry(entry, target.reason))
|
|
524
594
|
cancelled++;
|
|
@@ -530,7 +600,7 @@ export function createRequestRegistry(options) {
|
|
|
530
600
|
// Counted as one — unlike the pure race below, this targets a known
|
|
531
601
|
// in-flight (queued) begin, so the Stop button feels responsive
|
|
532
602
|
// instead of waiting for the slot to free first.
|
|
533
|
-
const queued = waitersById.get(
|
|
603
|
+
const queued = waitersById.get(registryKey);
|
|
534
604
|
if (queued) {
|
|
535
605
|
cancelQueuedWaiterGracefully(queued.key, queued.waiter, target.reason);
|
|
536
606
|
return cancelled + 1;
|
|
@@ -539,20 +609,30 @@ export function createRequestRegistry(options) {
|
|
|
539
609
|
// releasing its native context. There is nothing to cancel, and a
|
|
540
610
|
// cancel-before-begin marker would wrongly abort a later begin that
|
|
541
611
|
// reuses this id once teardown frees it.
|
|
542
|
-
if (disposingIds.has(
|
|
612
|
+
if (disposingIds.has(registryKey))
|
|
543
613
|
return cancelled;
|
|
544
614
|
// Stop-button race: the cancel beat its own
|
|
545
615
|
// `begin(...)`. Record the cancel so the next matching `begin`
|
|
546
616
|
// aborts immediately. The return value stays 0 — no in-flight
|
|
547
617
|
// request was matched, which is still the truth — but the
|
|
548
618
|
// *effective* cancel will land when the begin arrives.
|
|
549
|
-
recordCancelBeforeBegin(
|
|
619
|
+
recordCancelBeforeBegin(registryKey, target.reason);
|
|
550
620
|
return cancelled;
|
|
551
621
|
}
|
|
622
|
+
const matchesSession = (sessionId) => 'allModels' in target
|
|
623
|
+
? sessionId === target.sessionId
|
|
624
|
+
: target.allSessions === true || sessionId === (target.sessionId ?? OWNER_SESSION_ID);
|
|
625
|
+
const matchesWork = (modelId, kind) => {
|
|
626
|
+
if ('allModels' in target)
|
|
627
|
+
return true;
|
|
628
|
+
if (modelId !== target.modelId)
|
|
629
|
+
return false;
|
|
630
|
+
return !target.kind || kind === target.kind;
|
|
631
|
+
};
|
|
552
632
|
for (const entry of entries.values()) {
|
|
553
|
-
if (entry.ctx.
|
|
633
|
+
if (!matchesSession(entry.ctx.sessionId))
|
|
554
634
|
continue;
|
|
555
|
-
if (
|
|
635
|
+
if (!matchesWork(entry.ctx.modelId, entry.ctx.kind))
|
|
556
636
|
continue;
|
|
557
637
|
if (cancelEntry(entry, target.reason))
|
|
558
638
|
cancelled++;
|
|
@@ -562,9 +642,9 @@ export function createRequestRegistry(options) {
|
|
|
562
642
|
// strand them behind a request that's being cancelled out from under
|
|
563
643
|
// them. Snapshot first — `removeWaiter` mutates `waitersById`.
|
|
564
644
|
for (const { key, waiter } of Array.from(waitersById.values())) {
|
|
565
|
-
if (waiter.
|
|
645
|
+
if (!matchesSession(waiter.sessionId))
|
|
566
646
|
continue;
|
|
567
|
-
if (
|
|
647
|
+
if (!matchesWork(waiter.modelId, waiter.kind))
|
|
568
648
|
continue;
|
|
569
649
|
cancelQueuedWaiterGracefully(key, waiter, target.reason);
|
|
570
650
|
cancelled++;
|
|
@@ -625,8 +705,44 @@ export function createRequestRegistry(options) {
|
|
|
625
705
|
drainingModels.delete(modelId);
|
|
626
706
|
}
|
|
627
707
|
}
|
|
628
|
-
|
|
629
|
-
|
|
708
|
+
// Cancel every request a session owns and wait for the active ones to finish
|
|
709
|
+
// disposing. A begin still in admission, queued or granted but not yet
|
|
710
|
+
// landed, is marked cancelled first and waited for, so it lands as an aborted
|
|
711
|
+
// context before the active scan runs; its handler disposes it on its own and
|
|
712
|
+
// a model release after this goes through `withModelDraining`, which waits
|
|
713
|
+
// for those too.
|
|
714
|
+
async function drainSession(sessionId, reason) {
|
|
715
|
+
let cancelled = 0;
|
|
716
|
+
const landing = [];
|
|
717
|
+
for (const [key, pending] of pendingBegins) {
|
|
718
|
+
if (pending.sessionId !== sessionId)
|
|
719
|
+
continue;
|
|
720
|
+
const queued = waitersById.get(key);
|
|
721
|
+
if (queued)
|
|
722
|
+
cancelQueuedWaiterGracefully(queued.key, queued.waiter, reason);
|
|
723
|
+
else
|
|
724
|
+
recordCancelBeforeBegin(key, reason);
|
|
725
|
+
landing.push(pending.settled);
|
|
726
|
+
cancelled++;
|
|
727
|
+
}
|
|
728
|
+
await Promise.all(landing);
|
|
729
|
+
const draining = [];
|
|
730
|
+
for (const entry of entries.values()) {
|
|
731
|
+
if (entry.ctx.sessionId !== sessionId)
|
|
732
|
+
continue;
|
|
733
|
+
if (cancelEntry(entry, reason))
|
|
734
|
+
cancelled++;
|
|
735
|
+
draining.push(entry.disposed);
|
|
736
|
+
}
|
|
737
|
+
for (const entry of disposingEntries) {
|
|
738
|
+
if (entry.ctx.sessionId === sessionId)
|
|
739
|
+
draining.push(entry.disposed);
|
|
740
|
+
}
|
|
741
|
+
await Promise.all(draining);
|
|
742
|
+
return cancelled;
|
|
743
|
+
}
|
|
744
|
+
async function end(requestId, outcome, sessionId = OWNER_SESSION_ID) {
|
|
745
|
+
const entry = entries.get(requestRegistryKey(sessionId, requestId));
|
|
630
746
|
if (!entry)
|
|
631
747
|
return;
|
|
632
748
|
await disposeEntry(entry, outcome);
|
|
@@ -641,8 +757,10 @@ export function createRequestRegistry(options) {
|
|
|
641
757
|
cancel,
|
|
642
758
|
cancelAll,
|
|
643
759
|
withModelDraining,
|
|
760
|
+
drainSession,
|
|
644
761
|
end,
|
|
645
762
|
policy,
|
|
763
|
+
bindTransportSignal,
|
|
646
764
|
// Test-only: lets the registry race tests assert the bound
|
|
647
765
|
// invariants on the internal "cancelled-before-begin" set without
|
|
648
766
|
// exposing it as a public surface. Kept off the `RequestRegistry`
|
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
import { getEngineLogger } from '../logging/index.js';
|
|
2
2
|
import { LifecycleOperationBlockedError } from '../errors/index.js';
|
|
3
3
|
const logger = getEngineLogger();
|
|
4
|
-
|
|
4
|
+
// `closeSession` only releases local state (requests, vector indexes, model
|
|
5
|
+
// leases), so a session can end while the runtime is suspended. Refusing it
|
|
6
|
+
// would leave the session's resources held, since the client has already
|
|
7
|
+
// ended the handle.
|
|
8
|
+
const LIFECYCLE_ALLOWED_TYPES = new Set([
|
|
9
|
+
'suspend',
|
|
10
|
+
'resume',
|
|
11
|
+
'state',
|
|
12
|
+
'closeSession'
|
|
13
|
+
]);
|
|
5
14
|
const swarms = new Map();
|
|
6
15
|
const stores = new Map();
|
|
7
16
|
const resumeListeners = new Set();
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session id of the runtime owner: the process that started the engine and
|
|
3
|
+
* calls it without a session handle. Requests without a `sessionId` run as
|
|
4
|
+
* this session. It is never accepted on the wire (`openSession` refuses it and
|
|
5
|
+
* `assertWireSessionOpen` fails a request naming it), so a client cannot act
|
|
6
|
+
* as the owner by sending the string.
|
|
7
|
+
*/
|
|
8
|
+
export declare const OWNER_SESSION_ID = "owner";
|
|
9
|
+
/** The session a request runs as: its `sessionId`, or the owner when absent. */
|
|
10
|
+
export declare function resolveSessionId(request: {
|
|
11
|
+
sessionId?: string | undefined;
|
|
12
|
+
}): string;
|
|
13
|
+
/**
|
|
14
|
+
* Key the request registry stores a request under. The session id is length
|
|
15
|
+
* prefixed, so no choice of characters in either id lets two different
|
|
16
|
+
* `(sessionId, requestId)` pairs share a key.
|
|
17
|
+
*/
|
|
18
|
+
export declare function requestRegistryKey(sessionId: string, requestId: string): string;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session id of the runtime owner: the process that started the engine and
|
|
3
|
+
* calls it without a session handle. Requests without a `sessionId` run as
|
|
4
|
+
* this session. It is never accepted on the wire (`openSession` refuses it and
|
|
5
|
+
* `assertWireSessionOpen` fails a request naming it), so a client cannot act
|
|
6
|
+
* as the owner by sending the string.
|
|
7
|
+
*/
|
|
8
|
+
export const OWNER_SESSION_ID = 'owner';
|
|
9
|
+
/** The session a request runs as: its `sessionId`, or the owner when absent. */
|
|
10
|
+
export function resolveSessionId(request) {
|
|
11
|
+
return request.sessionId ?? OWNER_SESSION_ID;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Key the request registry stores a request under. The session id is length
|
|
15
|
+
* prefixed, so no choice of characters in either id lets two different
|
|
16
|
+
* `(sessionId, requestId)` pairs share a key.
|
|
17
|
+
*/
|
|
18
|
+
export function requestRegistryKey(sessionId, requestId) {
|
|
19
|
+
return `${sessionId.length}:${sessionId}:${requestId}`;
|
|
20
|
+
}
|