@qvac/inference 0.0.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +179 -0
- package/NOTICE +928 -0
- package/README.md +138 -0
- package/dist/api/audio-gen.d.ts +18 -0
- package/dist/api/audio-gen.js +148 -0
- package/dist/api/batch-completion.d.ts +14 -0
- package/dist/api/batch-completion.js +332 -0
- package/dist/api/bci-transcribe.d.ts +56 -0
- package/dist/api/bci-transcribe.js +172 -0
- package/dist/api/cancel.d.ts +34 -0
- package/dist/api/cancel.js +82 -0
- package/dist/api/classify.d.ts +22 -0
- package/dist/api/classify.js +46 -0
- package/dist/api/completion-stream.d.ts +106 -0
- package/dist/api/completion-stream.js +351 -0
- package/dist/api/delete-cache.d.ts +29 -0
- package/dist/api/delete-cache.js +49 -0
- package/dist/api/diffusion.d.ts +92 -0
- package/dist/api/diffusion.js +176 -0
- package/dist/api/download-asset.d.ts +49 -0
- package/dist/api/download-asset.js +87 -0
- package/dist/api/embed.d.ts +41 -0
- package/dist/api/embed.js +29 -0
- package/dist/api/finetune.d.ts +89 -0
- package/dist/api/finetune.js +138 -0
- package/dist/api/get-loaded-model-info.d.ts +24 -0
- package/dist/api/get-loaded-model-info.js +34 -0
- package/dist/api/get-model-info.d.ts +43 -0
- package/dist/api/get-model-info.js +21 -0
- package/dist/api/get-system-resources.d.ts +2 -0
- package/dist/api/get-system-resources.js +16 -0
- package/dist/api/heartbeat.d.ts +29 -0
- package/dist/api/heartbeat.js +37 -0
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +35 -0
- package/dist/api/invoke-plugin.d.ts +30 -0
- package/dist/api/invoke-plugin.js +54 -0
- package/dist/api/load-model.d.ts +169 -0
- package/dist/api/load-model.js +129 -0
- package/dist/api/logging-stream-registry.d.ts +4 -0
- package/dist/api/logging-stream-registry.js +61 -0
- package/dist/api/logging-stream.d.ts +23 -0
- package/dist/api/logging-stream.js +36 -0
- package/dist/api/ocr.d.ts +30 -0
- package/dist/api/ocr.js +88 -0
- package/dist/api/parse-input.d.ts +2 -0
- package/dist/api/parse-input.js +14 -0
- package/dist/api/provide.d.ts +23 -0
- package/dist/api/provide.js +32 -0
- package/dist/api/rag.d.ts +251 -0
- package/dist/api/rag.js +485 -0
- package/dist/api/registry.d.ts +39 -0
- package/dist/api/registry.js +63 -0
- package/dist/api/resume.d.ts +29 -0
- package/dist/api/resume.js +36 -0
- package/dist/api/state.d.ts +22 -0
- package/dist/api/state.js +29 -0
- package/dist/api/stop-provider.d.ts +20 -0
- package/dist/api/stop-provider.js +30 -0
- package/dist/api/subscribe-logs.d.ts +27 -0
- package/dist/api/subscribe-logs.js +43 -0
- package/dist/api/suspend.d.ts +30 -0
- package/dist/api/suspend.js +37 -0
- package/dist/api/text-to-speech.d.ts +64 -0
- package/dist/api/text-to-speech.js +434 -0
- package/dist/api/transcribe.d.ts +75 -0
- package/dist/api/transcribe.js +256 -0
- package/dist/api/translate.d.ts +47 -0
- package/dist/api/translate.js +115 -0
- package/dist/api/unload-model.d.ts +17 -0
- package/dist/api/unload-model.js +42 -0
- package/dist/api/upscale.d.ts +44 -0
- package/dist/api/upscale.js +99 -0
- package/dist/api/video.d.ts +86 -0
- package/dist/api/video.js +176 -0
- package/dist/api/vla-helpers.d.ts +43 -0
- package/dist/api/vla-helpers.js +124 -0
- package/dist/api/vla.d.ts +68 -0
- package/dist/api/vla.js +113 -0
- package/dist/config/config-utils.d.ts +4 -0
- package/dist/config/config-utils.js +18 -0
- package/dist/config/resolve-config.d.ts +8 -0
- package/dist/config/resolve-config.js +74 -0
- package/dist/constants/audio.d.ts +1 -0
- package/dist/constants/audio.js +1 -0
- package/dist/constants/index.d.ts +2 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/registry.d.ts +1 -0
- package/dist/constants/registry.js +1 -0
- package/dist/dispatch.d.ts +18 -0
- package/dist/dispatch.js +271 -0
- package/dist/engine.d.ts +5 -0
- package/dist/engine.js +27 -0
- package/dist/errors/index.d.ts +416 -0
- package/dist/errors/index.js +681 -0
- package/dist/errors/options.d.ts +2 -0
- package/dist/errors/options.js +13 -0
- package/dist/handlers/cancelHandler.d.ts +29 -0
- package/dist/handlers/cancelHandler.js +67 -0
- package/dist/handlers/completion-orchestrate.d.ts +41 -0
- package/dist/handlers/completion-orchestrate.js +190 -0
- package/dist/handlers/delete-cache.d.ts +11 -0
- package/dist/handlers/delete-cache.js +37 -0
- package/dist/handlers/download-asset.d.ts +2 -0
- package/dist/handlers/download-asset.js +146 -0
- package/dist/handlers/finetune.d.ts +2 -0
- package/dist/handlers/finetune.js +18 -0
- package/dist/handlers/get-loaded-model-info.d.ts +2 -0
- package/dist/handlers/get-loaded-model-info.js +45 -0
- package/dist/handlers/get-model-info.d.ts +2 -0
- package/dist/handlers/get-model-info.js +173 -0
- package/dist/handlers/get-system-resources.d.ts +2 -0
- package/dist/handlers/get-system-resources.js +53 -0
- package/dist/handlers/heartbeat.d.ts +2 -0
- package/dist/handlers/heartbeat.js +3 -0
- package/dist/handlers/load-model/download-manager.d.ts +95 -0
- package/dist/handlers/load-model/download-manager.js +306 -0
- package/dist/handlers/load-model/download-stats.d.ts +7 -0
- package/dist/handlers/load-model/download-stats.js +126 -0
- package/dist/handlers/load-model/handler.d.ts +2 -0
- package/dist/handlers/load-model/handler.js +287 -0
- package/dist/handlers/load-model/http.d.ts +3 -0
- package/dist/handlers/load-model/http.js +857 -0
- package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
- package/dist/handlers/load-model/hyperdrive.js +560 -0
- package/dist/handlers/load-model/index.d.ts +2 -0
- package/dist/handlers/load-model/index.js +2 -0
- package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/handlers/load-model/registry-client-options.js +18 -0
- package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/handlers/load-model/registry-companion-set.js +206 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
- package/dist/handlers/load-model/registry-download-utils.js +112 -0
- package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
- package/dist/handlers/load-model/registry-metadata.js +23 -0
- package/dist/handlers/load-model/registry.d.ts +4 -0
- package/dist/handlers/load-model/registry.js +298 -0
- package/dist/handlers/load-model/resolve-session.d.ts +33 -0
- package/dist/handlers/load-model/resolve-session.js +62 -0
- package/dist/handlers/load-model/resolve.d.ts +5 -0
- package/dist/handlers/load-model/resolve.js +158 -0
- package/dist/handlers/load-model/types.d.ts +59 -0
- package/dist/handlers/load-model/types.js +28 -0
- package/dist/handlers/logging-stream.d.ts +2 -0
- package/dist/handlers/logging-stream.js +39 -0
- package/dist/handlers/plugin-dispatch.d.ts +3 -0
- package/dist/handlers/plugin-dispatch.js +63 -0
- package/dist/handlers/plugin-invoke.d.ts +3 -0
- package/dist/handlers/plugin-invoke.js +81 -0
- package/dist/handlers/registry.d.ts +4 -0
- package/dist/handlers/registry.js +156 -0
- package/dist/handlers/resume.d.ts +2 -0
- package/dist/handlers/resume.js +11 -0
- package/dist/handlers/state.d.ts +2 -0
- package/dist/handlers/state.js +7 -0
- package/dist/handlers/suspend.d.ts +2 -0
- package/dist/handlers/suspend.js +11 -0
- package/dist/handlers/types.d.ts +13 -0
- package/dist/handlers/types.js +1 -0
- package/dist/handlers/unload-model.d.ts +2 -0
- package/dist/handlers/unload-model.js +29 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +12 -0
- package/dist/logging/addon.d.ts +5 -0
- package/dist/logging/addon.js +75 -0
- package/dist/logging/index.d.ts +5 -0
- package/dist/logging/index.js +4 -0
- package/dist/logging/logger.d.ts +41 -0
- package/dist/logging/logger.js +209 -0
- package/dist/logging/namespaces.d.ts +6 -0
- package/dist/logging/namespaces.js +7 -0
- package/dist/logging/types.d.ts +37 -0
- package/dist/logging/types.js +1 -0
- package/dist/logging/utils.d.ts +5 -0
- package/dist/logging/utils.js +113 -0
- package/dist/models/registry/index.d.ts +18 -0
- package/dist/models/registry/index.js +21 -0
- package/dist/models/registry/models.d.ts +22680 -0
- package/dist/models/registry/models.js +23908 -0
- package/dist/p2p/cancel-delegated.d.ts +3 -0
- package/dist/p2p/cancel-delegated.js +64 -0
- package/dist/p2p/completion-stream-delegated.d.ts +4 -0
- package/dist/p2p/completion-stream-delegated.js +51 -0
- package/dist/p2p/delegate-client.d.ts +9 -0
- package/dist/p2p/delegate-client.js +219 -0
- package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
- package/dist/p2p/delegate-connect-diagnostics.js +38 -0
- package/dist/p2p/delegate-transport.d.ts +18 -0
- package/dist/p2p/delegate-transport.js +237 -0
- package/dist/p2p/error-handlers.d.ts +4 -0
- package/dist/p2p/error-handlers.js +31 -0
- package/dist/p2p/handle-request.d.ts +2 -0
- package/dist/p2p/handle-request.js +171 -0
- package/dist/p2p/handler-utils.d.ts +19 -0
- package/dist/p2p/handler-utils.js +180 -0
- package/dist/p2p/heartbeat-delegated.d.ts +3 -0
- package/dist/p2p/heartbeat-delegated.js +32 -0
- package/dist/p2p/load-model-delegated.d.ts +6 -0
- package/dist/p2p/load-model-delegated.js +132 -0
- package/dist/p2p/progress-throttle.d.ts +7 -0
- package/dist/p2p/progress-throttle.js +47 -0
- package/dist/p2p/provideHandler/connection.d.ts +2 -0
- package/dist/p2p/provideHandler/connection.js +38 -0
- package/dist/p2p/provideHandler/handler.d.ts +2 -0
- package/dist/p2p/provideHandler/handler.js +66 -0
- package/dist/p2p/provideHandler/proxy.d.ts +2 -0
- package/dist/p2p/provideHandler/proxy.js +24 -0
- package/dist/p2p/provideHandler.d.ts +1 -0
- package/dist/p2p/provideHandler.js +2 -0
- package/dist/p2p/rpc-utils.d.ts +4 -0
- package/dist/p2p/rpc-utils.js +8 -0
- package/dist/p2p/stopProvideHandler.d.ts +2 -0
- package/dist/p2p/stopProvideHandler.js +29 -0
- package/dist/p2p/swarm.d.ts +9 -0
- package/dist/p2p/swarm.js +103 -0
- package/dist/p2p/transport-selector.d.ts +2 -0
- package/dist/p2p/transport-selector.js +18 -0
- package/dist/p2p/unload-model-delegated.d.ts +2 -0
- package/dist/p2p/unload-model-delegated.js +31 -0
- package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
- package/dist/plugins/builtin/asr-ggml/config.js +46 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
- package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
- package/dist/plugins/builtin/index.d.ts +11 -0
- package/dist/plugins/builtin/index.js +11 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
- package/dist/plugins/index.d.ts +9 -0
- package/dist/plugins/index.js +12 -0
- package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
- package/dist/plugins/ops/bci-transcribe.js +248 -0
- package/dist/plugins/ops/cancel.d.ts +25 -0
- package/dist/plugins/ops/cancel.js +43 -0
- package/dist/plugins/ops/embed.d.ts +15 -0
- package/dist/plugins/ops/embed.js +193 -0
- package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
- package/dist/plugins/ops/kv-cache-retention.js +121 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
- package/dist/plugins/ops/kv-cache-utils.js +120 -0
- package/dist/plugins/ops/load-model.d.ts +8 -0
- package/dist/plugins/ops/load-model.js +82 -0
- package/dist/plugins/ops/transcribe.d.ts +20 -0
- package/dist/plugins/ops/transcribe.js +343 -0
- package/dist/plugins/ops/translate.d.ts +7 -0
- package/dist/plugins/ops/translate.js +256 -0
- package/dist/plugins/ops/unload-model.d.ts +2 -0
- package/dist/plugins/ops/unload-model.js +45 -0
- package/dist/plugins/registry.d.ts +9 -0
- package/dist/plugins/registry.js +107 -0
- package/dist/profiling/aggregator.d.ts +14 -0
- package/dist/profiling/aggregator.js +100 -0
- package/dist/profiling/backend-diagnostics.d.ts +8 -0
- package/dist/profiling/backend-diagnostics.js +36 -0
- package/dist/profiling/clock.d.ts +8 -0
- package/dist/profiling/clock.js +23 -0
- package/dist/profiling/controller.d.ts +34 -0
- package/dist/profiling/controller.js +130 -0
- package/dist/profiling/delegation-profiler.d.ts +31 -0
- package/dist/profiling/delegation-profiler.js +155 -0
- package/dist/profiling/envelope.d.ts +10 -0
- package/dist/profiling/envelope.js +45 -0
- package/dist/profiling/events.d.ts +19 -0
- package/dist/profiling/events.js +35 -0
- package/dist/profiling/exporters.d.ts +9 -0
- package/dist/profiling/exporters.js +242 -0
- package/dist/profiling/index.d.ts +91 -0
- package/dist/profiling/index.js +87 -0
- package/dist/profiling/model-execution.d.ts +15 -0
- package/dist/profiling/model-execution.js +34 -0
- package/dist/profiling/operation-metrics.d.ts +24 -0
- package/dist/profiling/operation-metrics.js +370 -0
- package/dist/profiling/operation-wrappers.d.ts +17 -0
- package/dist/profiling/operation-wrappers.js +196 -0
- package/dist/profiling/profiler.d.ts +27 -0
- package/dist/profiling/profiler.js +127 -0
- package/dist/profiling/ring-buffer.d.ts +18 -0
- package/dist/profiling/ring-buffer.js +83 -0
- package/dist/profiling/types.d.ts +52 -0
- package/dist/profiling/types.js +1 -0
- package/dist/rag/handler.d.ts +2 -0
- package/dist/rag/handler.js +272 -0
- package/dist/rag/handlers/chunk.d.ts +2 -0
- package/dist/rag/handlers/chunk.js +7 -0
- package/dist/rag/handlers/close-workspace.d.ts +2 -0
- package/dist/rag/handlers/close-workspace.js +9 -0
- package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
- package/dist/rag/handlers/delete-embeddings.js +11 -0
- package/dist/rag/handlers/delete-workspace.d.ts +2 -0
- package/dist/rag/handlers/delete-workspace.js +14 -0
- package/dist/rag/handlers/ingest.d.ts +9 -0
- package/dist/rag/handlers/ingest.js +19 -0
- package/dist/rag/handlers/list-workspaces.d.ts +2 -0
- package/dist/rag/handlers/list-workspaces.js +4 -0
- package/dist/rag/handlers/reindex.d.ts +9 -0
- package/dist/rag/handlers/reindex.js +16 -0
- package/dist/rag/handlers/save-embeddings.d.ts +9 -0
- package/dist/rag/handlers/save-embeddings.js +15 -0
- package/dist/rag/handlers/search.d.ts +2 -0
- package/dist/rag/handlers/search.js +16 -0
- package/dist/rag/index.d.ts +11 -0
- package/dist/rag/index.js +11 -0
- package/dist/rag/rag-operation-manager.d.ts +28 -0
- package/dist/rag/rag-operation-manager.js +76 -0
- package/dist/rag/rag-workspace-manager.d.ts +14 -0
- package/dist/rag/rag-workspace-manager.js +143 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +166 -0
- package/dist/resources/collector.d.ts +643 -0
- package/dist/resources/collector.js +226 -0
- package/dist/resources/instance.d.ts +4 -0
- package/dist/resources/instance.js +15 -0
- package/dist/resources/native.d.ts +2 -0
- package/dist/resources/native.js +35 -0
- package/dist/resources/normalize.d.ts +12 -0
- package/dist/resources/normalize.js +73 -0
- package/dist/resources/types.d.ts +65 -0
- package/dist/resources/types.js +1 -0
- package/dist/runtime/cache-lock.d.ts +2 -0
- package/dist/runtime/cache-lock.js +67 -0
- package/dist/runtime/disposable-scope.d.ts +35 -0
- package/dist/runtime/disposable-scope.js +69 -0
- package/dist/runtime/env.d.ts +16 -0
- package/dist/runtime/env.js +53 -0
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +20 -0
- package/dist/runtime/lifecycle.js +102 -0
- package/dist/runtime/logging-stream-registry.d.ts +27 -0
- package/dist/runtime/logging-stream-registry.js +133 -0
- package/dist/runtime/model-config-utils.d.ts +14 -0
- package/dist/runtime/model-config-utils.js +128 -0
- package/dist/runtime/model-registry.d.ts +74 -0
- package/dist/runtime/model-registry.js +130 -0
- package/dist/runtime/registry-client.d.ts +4 -0
- package/dist/runtime/registry-client.js +118 -0
- package/dist/runtime/request-context.d.ts +72 -0
- package/dist/runtime/request-context.js +114 -0
- package/dist/runtime/request-id.d.ts +17 -0
- package/dist/runtime/request-id.js +29 -0
- package/dist/runtime/request-registry.d.ts +194 -0
- package/dist/runtime/request-registry.js +546 -0
- package/dist/runtime/runtime-lifecycle.d.ts +37 -0
- package/dist/runtime/runtime-lifecycle.js +163 -0
- package/dist/runtime/state.d.ts +26 -0
- package/dist/runtime/state.js +138 -0
- package/dist/schemas/audio-gen.d.ts +252 -0
- package/dist/schemas/audio-gen.js +69 -0
- package/dist/schemas/batch-completion-stream.d.ts +467 -0
- package/dist/schemas/batch-completion-stream.js +123 -0
- package/dist/schemas/bci-config.d.ts +74 -0
- package/dist/schemas/bci-config.js +58 -0
- package/dist/schemas/bci.d.ts +178 -0
- package/dist/schemas/bci.js +89 -0
- package/dist/schemas/cancel.d.ts +203 -0
- package/dist/schemas/cancel.js +135 -0
- package/dist/schemas/classification.d.ts +52 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +6420 -0
- package/dist/schemas/common.js +133 -0
- package/dist/schemas/completion-event.d.ts +248 -0
- package/dist/schemas/completion-event.js +94 -0
- package/dist/schemas/completion-stream.d.ts +475 -0
- package/dist/schemas/completion-stream.js +236 -0
- package/dist/schemas/config.d.ts +5022 -0
- package/dist/schemas/config.js +169 -0
- package/dist/schemas/delegate.d.ts +29 -0
- package/dist/schemas/delegate.js +40 -0
- package/dist/schemas/download-asset.d.ts +225 -0
- package/dist/schemas/download-asset.js +71 -0
- package/dist/schemas/embed.d.ts +41 -0
- package/dist/schemas/embed.js +35 -0
- package/dist/schemas/engine-addon-map.d.ts +20 -0
- package/dist/schemas/engine-addon-map.js +78 -0
- package/dist/schemas/error.d.ts +40 -0
- package/dist/schemas/error.js +65 -0
- package/dist/schemas/errors.d.ts +119 -0
- package/dist/schemas/errors.js +615 -0
- package/dist/schemas/finetune.d.ts +621 -0
- package/dist/schemas/finetune.js +163 -0
- package/dist/schemas/index.d.ts +45 -0
- package/dist/schemas/index.js +45 -0
- package/dist/schemas/json-schema.d.ts +15 -0
- package/dist/schemas/json-schema.js +2 -0
- package/dist/schemas/lifecycle.d.ts +38 -0
- package/dist/schemas/lifecycle.js +21 -0
- package/dist/schemas/llamacpp-config.d.ts +403 -0
- package/dist/schemas/llamacpp-config.js +133 -0
- package/dist/schemas/load-model.d.ts +25635 -0
- package/dist/schemas/load-model.js +629 -0
- package/dist/schemas/logging-stream.d.ts +33 -0
- package/dist/schemas/logging-stream.js +17 -0
- package/dist/schemas/mcp-adapter.d.ts +46 -0
- package/dist/schemas/mcp-adapter.js +1 -0
- package/dist/schemas/model-file.d.ts +27 -0
- package/dist/schemas/model-file.js +29 -0
- package/dist/schemas/model-info.d.ts +197 -0
- package/dist/schemas/model-info.js +144 -0
- package/dist/schemas/model-ops.d.ts +36 -0
- package/dist/schemas/model-ops.js +40 -0
- package/dist/schemas/model-src-utils.d.ts +192 -0
- package/dist/schemas/model-src-utils.js +61 -0
- package/dist/schemas/model-types.d.ts +275 -0
- package/dist/schemas/model-types.js +223 -0
- package/dist/schemas/ocr.d.ts +138 -0
- package/dist/schemas/ocr.js +59 -0
- package/dist/schemas/path-validation.d.ts +9 -0
- package/dist/schemas/path-validation.js +22 -0
- package/dist/schemas/plugin.d.ts +341 -0
- package/dist/schemas/plugin.js +241 -0
- package/dist/schemas/profiling.d.ts +695 -0
- package/dist/schemas/profiling.js +100 -0
- package/dist/schemas/provide.d.ts +50 -0
- package/dist/schemas/provide.js +32 -0
- package/dist/schemas/rag.d.ts +301 -0
- package/dist/schemas/rag.js +211 -0
- package/dist/schemas/registry.d.ts +453 -0
- package/dist/schemas/registry.js +104 -0
- package/dist/schemas/reload-config.d.ts +585 -0
- package/dist/schemas/reload-config.js +42 -0
- package/dist/schemas/runtime-context.d.ts +18 -0
- package/dist/schemas/runtime-context.js +7 -0
- package/dist/schemas/sdcpp-config.d.ts +1126 -0
- package/dist/schemas/sdcpp-config.js +748 -0
- package/dist/schemas/system-resources.d.ts +3966 -0
- package/dist/schemas/system-resources.js +149 -0
- package/dist/schemas/text-to-speech.d.ts +1352 -0
- package/dist/schemas/text-to-speech.js +364 -0
- package/dist/schemas/tools.d.ts +80 -0
- package/dist/schemas/tools.js +52 -0
- package/dist/schemas/transcription-config.d.ts +161 -0
- package/dist/schemas/transcription-config.js +156 -0
- package/dist/schemas/transcription.d.ts +261 -0
- package/dist/schemas/transcription.js +145 -0
- package/dist/schemas/translate.d.ts +138 -0
- package/dist/schemas/translate.js +103 -0
- package/dist/schemas/translation-config.d.ts +1031 -0
- package/dist/schemas/translation-config.js +170 -0
- package/dist/schemas/vla.d.ts +110 -0
- package/dist/schemas/vla.js +151 -0
- package/dist/selection.d.ts +8 -0
- package/dist/selection.js +14 -0
- package/dist/surface.d.ts +18 -0
- package/dist/surface.js +30 -0
- package/dist/utils/addon-responses.d.ts +58 -0
- package/dist/utils/addon-responses.js +1 -0
- package/dist/utils/aggregate-events.d.ts +25 -0
- package/dist/utils/aggregate-events.js +73 -0
- package/dist/utils/archive.d.ts +4 -0
- package/dist/utils/archive.js +180 -0
- package/dist/utils/asr-events.d.ts +14 -0
- package/dist/utils/asr-events.js +30 -0
- package/dist/utils/audio/decoder.d.ts +4 -0
- package/dist/utils/audio/decoder.js +80 -0
- package/dist/utils/audio-input.d.ts +9 -0
- package/dist/utils/audio-input.js +34 -0
- package/dist/utils/cache/auto-history.d.ts +3 -0
- package/dist/utils/cache/auto-history.js +26 -0
- package/dist/utils/cache/paths.d.ts +42 -0
- package/dist/utils/cache/paths.js +133 -0
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/cache/types.js +1 -0
- package/dist/utils/cache-normalize.d.ts +9 -0
- package/dist/utils/cache-normalize.js +20 -0
- package/dist/utils/checksum.d.ts +3 -0
- package/dist/utils/checksum.js +29 -0
- package/dist/utils/clear-storage.d.ts +4 -0
- package/dist/utils/clear-storage.js +14 -0
- package/dist/utils/completion-normalizer.d.ts +17 -0
- package/dist/utils/completion-normalizer.js +453 -0
- package/dist/utils/config-transform.d.ts +2 -0
- package/dist/utils/config-transform.js +21 -0
- package/dist/utils/decorate-promise.d.ts +33 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/encoding.d.ts +16 -0
- package/dist/utils/encoding.js +27 -0
- package/dist/utils/expand-gguf-shards.d.ts +2 -0
- package/dist/utils/expand-gguf-shards.js +22 -0
- package/dist/utils/formatting.d.ts +18 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
- package/dist/utils/gguf-tensor-extractor.js +311 -0
- package/dist/utils/global-singleton.d.ts +1 -0
- package/dist/utils/global-singleton.js +10 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/load-model-validation.d.ts +6 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/dist/utils/mcp-adapter.d.ts +11 -0
- package/dist/utils/mcp-adapter.js +96 -0
- package/dist/utils/multi-gpu-mobile.d.ts +4 -0
- package/dist/utils/multi-gpu-mobile.js +10 -0
- package/dist/utils/object.d.ts +1 -0
- package/dist/utils/object.js +3 -0
- package/dist/utils/path-security.d.ts +22 -0
- package/dist/utils/path-security.js +69 -0
- package/dist/utils/qvac-paths.d.ts +1 -0
- package/dist/utils/qvac-paths.js +5 -0
- package/dist/utils/response-format.d.ts +2 -0
- package/dist/utils/response-format.js +20 -0
- package/dist/utils/shard-utils.d.ts +47 -0
- package/dist/utils/shard-utils.js +123 -0
- package/dist/utils/tool-helpers.d.ts +24 -0
- package/dist/utils/tool-helpers.js +111 -0
- package/dist/utils/tool-integration.d.ts +23 -0
- package/dist/utils/tool-integration.js +29 -0
- package/dist/utils/tools/dialect.d.ts +7 -0
- package/dist/utils/tools/dialect.js +20 -0
- package/dist/utils/tools/index.d.ts +2 -0
- package/dist/utils/tools/index.js +2 -0
- package/dist/utils/tools/parser.d.ts +5 -0
- package/dist/utils/tools/parser.js +69 -0
- package/dist/utils/tools/parsers/dsml.d.ts +3 -0
- package/dist/utils/tools/parsers/dsml.js +177 -0
- package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
- package/dist/utils/tools/parsers/gemma4native.js +65 -0
- package/dist/utils/tools/parsers/harmony.d.ts +3 -0
- package/dist/utils/tools/parsers/harmony.js +57 -0
- package/dist/utils/tools/parsers/hermes.d.ts +3 -0
- package/dist/utils/tools/parsers/hermes.js +122 -0
- package/dist/utils/tools/parsers/json.d.ts +5 -0
- package/dist/utils/tools/parsers/json.js +115 -0
- package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
- package/dist/utils/tools/parsers/pythonic.js +267 -0
- package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
- package/dist/utils/tools/parsers/qwen35.js +125 -0
- package/dist/utils/tools/shared.d.ts +17 -0
- package/dist/utils/tools/shared.js +52 -0
- package/dist/utils/transcribe-metadata.d.ts +20 -0
- package/dist/utils/transcribe-metadata.js +28 -0
- package/dist/utils/tts-stats.d.ts +25 -0
- package/dist/utils/tts-stats.js +16 -0
- package/dist/utils/withTimeout.d.ts +17 -0
- package/dist/utils/withTimeout.js +47 -0
- package/dist/utils/zod-error.d.ts +2 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +266 -1
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { ModelAlreadyRegisteredError, ModelNotFoundError, ModelIsDelegatedError } from '../errors/index.js';
|
|
2
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
3
|
+
const logger = getEngineLogger();
|
|
4
|
+
// Global registry state - using stateless functions to manage it
|
|
5
|
+
const modelRegistry = new Map();
|
|
6
|
+
export function registerModel(id, options) {
|
|
7
|
+
if (modelRegistry.has(id)) {
|
|
8
|
+
throw new ModelAlreadyRegisteredError(id);
|
|
9
|
+
}
|
|
10
|
+
const isDelegated = 'providerPublicKey' in options;
|
|
11
|
+
if (isDelegated) {
|
|
12
|
+
const { providerPublicKey, timeout, healthCheckTimeout } = options;
|
|
13
|
+
modelRegistry.set(id, {
|
|
14
|
+
id,
|
|
15
|
+
isDelegated: true,
|
|
16
|
+
delegated: {
|
|
17
|
+
providerPublicKey,
|
|
18
|
+
timeout,
|
|
19
|
+
healthCheckTimeout
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
logger.info(`Delegated model registered: ${id} -> provider: ${providerPublicKey}, timeout: ${timeout}ms`);
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
modelRegistry.set(id, {
|
|
26
|
+
id,
|
|
27
|
+
isDelegated: false,
|
|
28
|
+
local: {
|
|
29
|
+
model: options.model,
|
|
30
|
+
path: options.path,
|
|
31
|
+
loadedAt: new Date(),
|
|
32
|
+
config: options.config,
|
|
33
|
+
modelType: options.modelType,
|
|
34
|
+
name: options.name
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
const nameStr = options.name ? ` (${options.name})` : '';
|
|
38
|
+
logger.info(`Local model registered: ${id}${nameStr} -> ${options.path}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export function getModelEntry(id) {
|
|
42
|
+
return modelRegistry.get(id) || null;
|
|
43
|
+
}
|
|
44
|
+
export function getModel(id) {
|
|
45
|
+
const entry = modelRegistry.get(id);
|
|
46
|
+
if (!entry) {
|
|
47
|
+
throw new ModelNotFoundError(id);
|
|
48
|
+
}
|
|
49
|
+
if (entry.isDelegated) {
|
|
50
|
+
throw new ModelIsDelegatedError(id);
|
|
51
|
+
}
|
|
52
|
+
return entry.local.model;
|
|
53
|
+
}
|
|
54
|
+
export function isModelLoaded(id) {
|
|
55
|
+
return modelRegistry.has(id);
|
|
56
|
+
}
|
|
57
|
+
export function unregisterModel(id) {
|
|
58
|
+
const entry = modelRegistry.get(id);
|
|
59
|
+
if (entry) {
|
|
60
|
+
modelRegistry.delete(id);
|
|
61
|
+
logger.debug(`Model unregistered: ${id}`);
|
|
62
|
+
return entry;
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
export function getAllModelIds() {
|
|
67
|
+
return Array.from(modelRegistry.keys());
|
|
68
|
+
}
|
|
69
|
+
export function getModelInfo(id) {
|
|
70
|
+
const entry = modelRegistry.get(id);
|
|
71
|
+
if (!entry || entry.isDelegated) {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
const result = {
|
|
75
|
+
id: entry.id,
|
|
76
|
+
path: entry.local.path,
|
|
77
|
+
loadedAt: entry.local.loadedAt,
|
|
78
|
+
config: entry.local.config
|
|
79
|
+
};
|
|
80
|
+
if (entry.local.name) {
|
|
81
|
+
result.name = entry.local.name;
|
|
82
|
+
}
|
|
83
|
+
return result;
|
|
84
|
+
}
|
|
85
|
+
export function getModelConfig(id) {
|
|
86
|
+
const entry = modelRegistry.get(id);
|
|
87
|
+
if (!entry || entry.isDelegated) {
|
|
88
|
+
throw new ModelNotFoundError(id);
|
|
89
|
+
}
|
|
90
|
+
return entry.local.config;
|
|
91
|
+
}
|
|
92
|
+
export function updateModelConfig(id, config) {
|
|
93
|
+
const entry = modelRegistry.get(id);
|
|
94
|
+
if (!entry) {
|
|
95
|
+
throw new ModelNotFoundError(id);
|
|
96
|
+
}
|
|
97
|
+
if (entry.isDelegated) {
|
|
98
|
+
throw new ModelIsDelegatedError(id);
|
|
99
|
+
}
|
|
100
|
+
entry.local.config = config;
|
|
101
|
+
}
|
|
102
|
+
export function clearRegistry() {
|
|
103
|
+
modelRegistry.clear();
|
|
104
|
+
logger.info('Model registry cleared');
|
|
105
|
+
}
|
|
106
|
+
export function getRegistryStats() {
|
|
107
|
+
return {
|
|
108
|
+
totalModels: modelRegistry.size,
|
|
109
|
+
modelIds: Array.from(modelRegistry.keys())
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
export async function unloadAllModels() {
|
|
113
|
+
const modelIds = getAllModelIds();
|
|
114
|
+
for (const modelId of modelIds) {
|
|
115
|
+
const entry = modelRegistry.get(modelId);
|
|
116
|
+
try {
|
|
117
|
+
if (entry && !entry.isDelegated) {
|
|
118
|
+
if (entry.local.model.unload) {
|
|
119
|
+
await entry.local.model.unload();
|
|
120
|
+
}
|
|
121
|
+
logger.debug(`Model unloaded: ${modelId}`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
logger.error(`Error unloading model ${modelId}:`, error instanceof Error ? error.message : String(error));
|
|
126
|
+
}
|
|
127
|
+
modelRegistry.delete(modelId);
|
|
128
|
+
}
|
|
129
|
+
logger.info(`Unloaded ${modelIds.length} models`);
|
|
130
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { QVACRegistryClient } from '@qvac/registry-client';
|
|
2
|
+
import { getCacheDir } from '../utils/cache/paths.js';
|
|
3
|
+
import { registerSwarm, unregisterSwarm, registerCorestore, unregisterCorestore } from '../runtime/runtime-lifecycle.js';
|
|
4
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
5
|
+
import { DEFAULT_REGISTRY_CORE_KEY } from '../constants/index.js';
|
|
6
|
+
const logger = getEngineLogger();
|
|
7
|
+
// fd-lock retry budget for the registry corestore.
|
|
8
|
+
//
|
|
9
|
+
// We pass `corestoreOpts: { wait: false }` (tryLock semantics) and provide a
|
|
10
|
+
// JS-bounded retry budget here instead of letting Hypercore's underlying
|
|
11
|
+
// fd-lock issue a blocking flock(LOCK_EX) on a libuv worker thread. The
|
|
12
|
+
// blocking variant cannot be cancelled from JS, so if another QVAC process
|
|
13
|
+
// holds the lock indefinitely it leaves a pending native handle that
|
|
14
|
+
// prevents Bare.exit() from terminating the process — see QVAC-18197.
|
|
15
|
+
//
|
|
16
|
+
// With tryLock + bounded retries we get the same "tolerate transient locks
|
|
17
|
+
// during another QVAC process's startup/shutdown" property #1480 wanted, but every
|
|
18
|
+
// retry step is a fresh non-blocking syscall that surfaces failure to JS,
|
|
19
|
+
// so shutdown always remains cancellable.
|
|
20
|
+
const MAX_RETRIES = 8;
|
|
21
|
+
const BASE_DELAY_MS = 250;
|
|
22
|
+
const MAX_TOTAL_WAIT_MS = 10_000;
|
|
23
|
+
let registryClient = null;
|
|
24
|
+
let inflightInit = null;
|
|
25
|
+
function isFdLockError(error) {
|
|
26
|
+
if (error instanceof Error) {
|
|
27
|
+
return error.message.includes('File descriptor could not be locked');
|
|
28
|
+
}
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
async function delay(ms) {
|
|
32
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
33
|
+
}
|
|
34
|
+
async function initRegistryClient() {
|
|
35
|
+
let lastError;
|
|
36
|
+
const deadline = Date.now() + MAX_TOTAL_WAIT_MS;
|
|
37
|
+
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
|
38
|
+
try {
|
|
39
|
+
const client = new QVACRegistryClient({
|
|
40
|
+
registryCoreKey: DEFAULT_REGISTRY_CORE_KEY,
|
|
41
|
+
storage: getCacheDir(`registry-corestore/${DEFAULT_REGISTRY_CORE_KEY}`),
|
|
42
|
+
corestoreOpts: { wait: false }
|
|
43
|
+
});
|
|
44
|
+
await client.ready();
|
|
45
|
+
registryClient = client;
|
|
46
|
+
if (registryClient.corestore) {
|
|
47
|
+
registerCorestore(registryClient.corestore, {
|
|
48
|
+
label: 'registry-client',
|
|
49
|
+
createdAt: Date.now()
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
if (registryClient.hyperswarm) {
|
|
53
|
+
registerSwarm(registryClient.hyperswarm, {
|
|
54
|
+
label: 'registry-client',
|
|
55
|
+
createdAt: Date.now()
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
logger.info('✅ Registry client ready');
|
|
59
|
+
return client;
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
lastError = error;
|
|
63
|
+
if (isFdLockError(error) && attempt < MAX_RETRIES) {
|
|
64
|
+
const remaining = deadline - Date.now();
|
|
65
|
+
if (remaining <= 0) {
|
|
66
|
+
logger.warn(`Registry client fd-lock failed after ${MAX_TOTAL_WAIT_MS}ms wait budget exhausted (attempt ${attempt}/${MAX_RETRIES})`);
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
const backoff = Math.min(BASE_DELAY_MS * Math.pow(2, attempt - 1), remaining);
|
|
70
|
+
logger.warn(`Registry client fd-lock failed (attempt ${attempt}/${MAX_RETRIES}), retrying in ${backoff}ms...`);
|
|
71
|
+
await delay(backoff);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
throw error;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
throw lastError;
|
|
78
|
+
}
|
|
79
|
+
export async function getRegistryClient() {
|
|
80
|
+
if (registryClient) {
|
|
81
|
+
logger.debug('Registry client reused');
|
|
82
|
+
return registryClient;
|
|
83
|
+
}
|
|
84
|
+
if (inflightInit) {
|
|
85
|
+
logger.debug('Registry client init already in progress, waiting...');
|
|
86
|
+
return inflightInit;
|
|
87
|
+
}
|
|
88
|
+
logger.info('🔗 Creating new registry client...');
|
|
89
|
+
inflightInit = initRegistryClient().finally(() => {
|
|
90
|
+
inflightInit = null;
|
|
91
|
+
});
|
|
92
|
+
return inflightInit;
|
|
93
|
+
}
|
|
94
|
+
export async function closeRegistryClient() {
|
|
95
|
+
if (!registryClient)
|
|
96
|
+
return;
|
|
97
|
+
const client = registryClient;
|
|
98
|
+
registryClient = null;
|
|
99
|
+
const corestore = client.corestore;
|
|
100
|
+
const hyperswarm = client.hyperswarm;
|
|
101
|
+
logger.info('🔌 Closing registry client...');
|
|
102
|
+
try {
|
|
103
|
+
await client.close();
|
|
104
|
+
logger.info('✅ Registry client closed');
|
|
105
|
+
}
|
|
106
|
+
catch (error) {
|
|
107
|
+
logger.error('❌ Error closing registry client:', error instanceof Error ? error.message : String(error));
|
|
108
|
+
}
|
|
109
|
+
finally {
|
|
110
|
+
if (corestore)
|
|
111
|
+
unregisterCorestore(corestore);
|
|
112
|
+
if (hyperswarm)
|
|
113
|
+
unregisterSwarm(hyperswarm);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
export function hasActiveRegistryClient() {
|
|
117
|
+
return registryClient !== null;
|
|
118
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { DisposableScope } from '../runtime/disposable-scope.js';
|
|
3
|
+
import { createRequestRegistry as createRegistry, type RequestRegistry } from '../runtime/request-registry.js';
|
|
4
|
+
import type { Logger } from '../logging/types.js';
|
|
5
|
+
/**
|
|
6
|
+
* Coarse classification of a long-running request. Used by
|
|
7
|
+
* `RequestRegistry.cancel({ modelId, kind })` so a broad cancel can target
|
|
8
|
+
* just one operation kind on a given model (e.g. cancel an in-flight
|
|
9
|
+
* completion without touching a finetune running on the same model).
|
|
10
|
+
*
|
|
11
|
+
* The set is intentionally open-coded — adding a new kind is a one-line
|
|
12
|
+
* change and the union surfaces in editor autocomplete at every call site.
|
|
13
|
+
*/
|
|
14
|
+
export type RequestKind = 'completion' | 'batchCompletion' | 'embeddings' | 'transcribe' | 'translate' | 'diffusion' | 'audiogen' | 'tts' | 'ocr' | 'vla' | 'finetune' | 'loadModel' | 'downloadAsset' | 'rag';
|
|
15
|
+
/**
|
|
16
|
+
* Lifecycle states a request transitions through. A new context starts in
|
|
17
|
+
* `"running"`. `cancel(...)` flips it to `"cancelling"` and aborts the
|
|
18
|
+
* signal; `end({ outcome: "completed" | "failed" | "cancelled" })` flips
|
|
19
|
+
* it to a terminal state and removes it from the registry.
|
|
20
|
+
*
|
|
21
|
+
* Kept as a string union (not a state machine) on purpose — handlers read
|
|
22
|
+
* `state` defensively at most a couple of points and a flat enum is easier
|
|
23
|
+
* to log/assert than a transition table.
|
|
24
|
+
*/
|
|
25
|
+
export type RequestState = 'running' | 'cancelling' | 'completed' | 'failed' | 'cancelled';
|
|
26
|
+
/**
|
|
27
|
+
* Per-request lifecycle handle. Created by `RequestRegistry.begin(...)`
|
|
28
|
+
* and consumed by long-running handlers as the single owner of:
|
|
29
|
+
*
|
|
30
|
+
* - `requestId` — stable identity; visible to the caller so it can
|
|
31
|
+
* target this exact request with `cancel({ requestId })`.
|
|
32
|
+
* - `signal` — `AbortSignal` that fires when the request is cancelled.
|
|
33
|
+
* Composes with addon-level cancellation through a single
|
|
34
|
+
* `signal.addEventListener("abort", ...)` hook installed by the
|
|
35
|
+
* handler.
|
|
36
|
+
* - `scope` — `DisposableScope` for `await using` / `Symbol.asyncDispose`
|
|
37
|
+
* cleanup. The scope unwinds whether the handler returns, throws, or
|
|
38
|
+
* is cancelled — there is no manual cleanup path for handlers to
|
|
39
|
+
* forget on the cancel branch.
|
|
40
|
+
* - `state` — current lifecycle state. Treat as read-mostly; the
|
|
41
|
+
* registry mutates it.
|
|
42
|
+
*/
|
|
43
|
+
export interface RequestContext {
|
|
44
|
+
readonly requestId: string;
|
|
45
|
+
readonly kind: RequestKind;
|
|
46
|
+
readonly modelId: string | undefined;
|
|
47
|
+
readonly signal: AbortSignal;
|
|
48
|
+
readonly scope: DisposableScope;
|
|
49
|
+
state: RequestState;
|
|
50
|
+
}
|
|
51
|
+
export declare function getRequestRegistry(): RequestRegistry;
|
|
52
|
+
export { createRegistry as createRequestRegistry };
|
|
53
|
+
export interface RequestLogContext {
|
|
54
|
+
requestId: string;
|
|
55
|
+
kind: string;
|
|
56
|
+
modelId: string | undefined;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Wraps `logger` so every emit is prefixed with
|
|
60
|
+
* `[request-lifecycle <kind> requestId=<id> modelId=<modelId>] `
|
|
61
|
+
* (the `modelId=...` segment is dropped when absent). The wrapper is a
|
|
62
|
+
* thin shim: `setLevel` / `getLevel` / `addTransport` / `setConsoleOutput`
|
|
63
|
+
* pass through to the underlying logger, and transport callbacks receive
|
|
64
|
+
* the prefixed message.
|
|
65
|
+
*
|
|
66
|
+
* @example
|
|
67
|
+
* await using ctx = await registry.begin({ requestId, kind: "completion", modelId });
|
|
68
|
+
* const log = withRequestContext(getEngineLogger(), ctx);
|
|
69
|
+
* log.info("decoding token 7");
|
|
70
|
+
* // → "[request-lifecycle completion requestId=<id> modelId=<id>] decoding token 7"
|
|
71
|
+
*/
|
|
72
|
+
export declare function withRequestContext(logger: Logger, ctx: RequestLogContext): Logger;
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { createRequestRegistry as createRegistry } from '../runtime/request-registry.js';
|
|
2
|
+
/**
|
|
3
|
+
* Shared singleton. Every long-running request registers under this
|
|
4
|
+
* registry, so a `cancel({ requestId })` call can find its target without
|
|
5
|
+
* the caller needing to know which plugin / handler owns the request.
|
|
6
|
+
*
|
|
7
|
+
* Exposed alongside `createRequestRegistry()` rather than replacing it so
|
|
8
|
+
* unit tests can spin up isolated registries without contaminating the
|
|
9
|
+
* shared instance. On first use the singleton registers the engine's
|
|
10
|
+
* baseline concurrency policies.
|
|
11
|
+
*/
|
|
12
|
+
let registry = null;
|
|
13
|
+
// `completion` and `batchCompletion` both run on the same `@qvac/llm-llamacpp`
|
|
14
|
+
// instance, which funnels every `run()` (single-prompt and batch alike) through
|
|
15
|
+
// one per-instance exclusive run queue plus a single-job native runner. They
|
|
16
|
+
// therefore can't actually execute at once on the same model. Sharing one
|
|
17
|
+
// admission lane makes the queue reflect that reality: a completion and a batch
|
|
18
|
+
// on the same model serialize FIFO at the registry layer instead of both being
|
|
19
|
+
// admitted and silently serializing inside the addon (which hides them from the
|
|
20
|
+
// registry's queue-depth accounting, `requestId` diagnostics, and cancel).
|
|
21
|
+
const LLAMACPP_COMPLETION_SLOT_GROUP = 'llamacppCompletion';
|
|
22
|
+
function installDefaultPolicies(r) {
|
|
23
|
+
// A loaded model is a single native context (one KV-cache, single-slot
|
|
24
|
+
// decode), so two same-model completions can't run in parallel. Serialize
|
|
25
|
+
// rather than reject: the second waits FIFO. maxConcurrentPerModel: 1 is
|
|
26
|
+
// today's reality — raise it once continuous batching lands. The depth cap
|
|
27
|
+
// bounds queue memory. The shared slot group extends that serialization
|
|
28
|
+
// across `completion` + `batchCompletion` on the same model (see note above).
|
|
29
|
+
r.policy({
|
|
30
|
+
kind: 'completion',
|
|
31
|
+
maxConcurrentPerModel: 1,
|
|
32
|
+
onOverflow: 'queue',
|
|
33
|
+
maxQueueDepthPerModel: 64,
|
|
34
|
+
sharedSlotGroup: LLAMACPP_COMPLETION_SLOT_GROUP
|
|
35
|
+
});
|
|
36
|
+
r.policy({
|
|
37
|
+
kind: 'batchCompletion',
|
|
38
|
+
maxConcurrentPerModel: 1,
|
|
39
|
+
onOverflow: 'queue',
|
|
40
|
+
maxQueueDepthPerModel: 64,
|
|
41
|
+
sharedSlotGroup: LLAMACPP_COMPLETION_SLOT_GROUP
|
|
42
|
+
});
|
|
43
|
+
// ACE-Step owns one active job per model. Starting another run replaces the
|
|
44
|
+
// addon's active response, and model-scoped cancel targets that single active
|
|
45
|
+
// job. Keep the registry authoritative by admitting one AudioGen request per
|
|
46
|
+
// model and queueing later requests FIFO.
|
|
47
|
+
r.policy({
|
|
48
|
+
kind: 'audiogen',
|
|
49
|
+
maxConcurrentPerModel: 1,
|
|
50
|
+
onOverflow: 'queue',
|
|
51
|
+
maxQueueDepthPerModel: 64
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
export function getRequestRegistry() {
|
|
55
|
+
if (!registry) {
|
|
56
|
+
registry = createRegistry();
|
|
57
|
+
installDefaultPolicies(registry);
|
|
58
|
+
}
|
|
59
|
+
return registry;
|
|
60
|
+
}
|
|
61
|
+
export { createRegistry as createRequestRegistry };
|
|
62
|
+
/**
|
|
63
|
+
* Wraps `logger` so every emit is prefixed with
|
|
64
|
+
* `[request-lifecycle <kind> requestId=<id> modelId=<modelId>] `
|
|
65
|
+
* (the `modelId=...` segment is dropped when absent). The wrapper is a
|
|
66
|
+
* thin shim: `setLevel` / `getLevel` / `addTransport` / `setConsoleOutput`
|
|
67
|
+
* pass through to the underlying logger, and transport callbacks receive
|
|
68
|
+
* the prefixed message.
|
|
69
|
+
*
|
|
70
|
+
* @example
|
|
71
|
+
* await using ctx = await registry.begin({ requestId, kind: "completion", modelId });
|
|
72
|
+
* const log = withRequestContext(getEngineLogger(), ctx);
|
|
73
|
+
* log.info("decoding token 7");
|
|
74
|
+
* // → "[request-lifecycle completion requestId=<id> modelId=<id>] decoding token 7"
|
|
75
|
+
*/
|
|
76
|
+
export function withRequestContext(logger, ctx) {
|
|
77
|
+
const prefix = ctx.modelId !== undefined
|
|
78
|
+
? `[request-lifecycle ${ctx.kind} requestId=${ctx.requestId} modelId=${ctx.modelId}] `
|
|
79
|
+
: `[request-lifecycle ${ctx.kind} requestId=${ctx.requestId}] `;
|
|
80
|
+
function pick(method) {
|
|
81
|
+
switch (method) {
|
|
82
|
+
case 'error':
|
|
83
|
+
return logger.error;
|
|
84
|
+
case 'warn':
|
|
85
|
+
return logger.warn;
|
|
86
|
+
case 'info':
|
|
87
|
+
return logger.info;
|
|
88
|
+
case 'debug':
|
|
89
|
+
return logger.debug;
|
|
90
|
+
case 'trace':
|
|
91
|
+
return logger.trace;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
function emit(method, args) {
|
|
95
|
+
const sink = pick(method);
|
|
96
|
+
if (args.length === 0) {
|
|
97
|
+
sink(prefix);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
const [first, ...rest] = args;
|
|
101
|
+
sink(prefix + String(first), ...rest);
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
error: (...args) => emit('error', args),
|
|
105
|
+
warn: (...args) => emit('warn', args),
|
|
106
|
+
info: (...args) => emit('info', args),
|
|
107
|
+
debug: (...args) => emit('debug', args),
|
|
108
|
+
trace: (...args) => emit('trace', args),
|
|
109
|
+
setLevel: (level) => logger.setLevel(level),
|
|
110
|
+
getLevel: () => logger.getLevel(),
|
|
111
|
+
addTransport: (transport) => logger.addTransport(transport),
|
|
112
|
+
setConsoleOutput: (enabled) => logger.setConsoleOutput(enabled)
|
|
113
|
+
};
|
|
114
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mint the `requestId` at call time so it can be surfaced synchronously on the
|
|
3
|
+
* returned handle (e.g. the `CompletionRun`) for use with `cancel({ requestId })`.
|
|
4
|
+
* The decorated promise's `op.requestId` and the request registry entry's
|
|
5
|
+
* `requestId` must be the same value, so it is generated exactly once here.
|
|
6
|
+
*
|
|
7
|
+
* Prefers `crypto.randomUUID` and falls back to 128 random bits as hex when it
|
|
8
|
+
* is unavailable — the fallback keeps `requestId` semantics (unique, opaque)
|
|
9
|
+
* without guaranteeing the UUIDv4 format.
|
|
10
|
+
*/
|
|
11
|
+
export declare function generateRequestId(): string;
|
|
12
|
+
/**
|
|
13
|
+
* Fill in a missing `requestId` from the runtime's UUID generator. The request
|
|
14
|
+
* schema marks `requestId` optional; this supplies a v4 UUID when the caller
|
|
15
|
+
* did not provide one.
|
|
16
|
+
*/
|
|
17
|
+
export declare function generateRandomRequestId(): string;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { randomUUID } from 'bare-crypto';
|
|
2
|
+
/**
|
|
3
|
+
* Mint the `requestId` at call time so it can be surfaced synchronously on the
|
|
4
|
+
* returned handle (e.g. the `CompletionRun`) for use with `cancel({ requestId })`.
|
|
5
|
+
* The decorated promise's `op.requestId` and the request registry entry's
|
|
6
|
+
* `requestId` must be the same value, so it is generated exactly once here.
|
|
7
|
+
*
|
|
8
|
+
* Prefers `crypto.randomUUID` and falls back to 128 random bits as hex when it
|
|
9
|
+
* is unavailable — the fallback keeps `requestId` semantics (unique, opaque)
|
|
10
|
+
* without guaranteeing the UUIDv4 format.
|
|
11
|
+
*/
|
|
12
|
+
export function generateRequestId() {
|
|
13
|
+
const c = globalThis.crypto;
|
|
14
|
+
if (c?.randomUUID)
|
|
15
|
+
return c.randomUUID();
|
|
16
|
+
const bytes = new Uint8Array(16);
|
|
17
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
18
|
+
bytes[i] = Math.floor(Math.random() * 256);
|
|
19
|
+
}
|
|
20
|
+
return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('');
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Fill in a missing `requestId` from the runtime's UUID generator. The request
|
|
24
|
+
* schema marks `requestId` optional; this supplies a v4 UUID when the caller
|
|
25
|
+
* did not provide one.
|
|
26
|
+
*/
|
|
27
|
+
export function generateRandomRequestId() {
|
|
28
|
+
return randomUUID();
|
|
29
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { RequestContext, RequestKind } from '../runtime/request-context.js';
|
|
3
|
+
import type { Logger } from '../logging/types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Outcome the caller declares when terminating a request through
|
|
6
|
+
* `registry.end(...)`. The registry maps it to a terminal `RequestState`
|
|
7
|
+
* before disposing the scope so observers see a coherent final state.
|
|
8
|
+
*/
|
|
9
|
+
export type RequestOutcome = 'completed' | 'failed' | 'cancelled';
|
|
10
|
+
export interface BeginOpts {
|
|
11
|
+
/** Stable identity. Caller-provided so a `cancel` can target it by id. */
|
|
12
|
+
requestId: string;
|
|
13
|
+
kind: RequestKind;
|
|
14
|
+
modelId?: string;
|
|
15
|
+
/**
|
|
16
|
+
* Optional parent abort signal — typically the process-level "shutdown"
|
|
17
|
+
* signal. When the parent aborts, the request's own signal aborts too.
|
|
18
|
+
* Composes through a `addEventListener("abort", ...)` hook so cancelling
|
|
19
|
+
* the parent does not require iterating the registry.
|
|
20
|
+
*/
|
|
21
|
+
parentSignal?: AbortSignal;
|
|
22
|
+
}
|
|
23
|
+
export interface CancelByRequestId {
|
|
24
|
+
requestId: string;
|
|
25
|
+
reason?: string;
|
|
26
|
+
}
|
|
27
|
+
export interface CancelByModelId {
|
|
28
|
+
modelId: string;
|
|
29
|
+
kind?: RequestKind;
|
|
30
|
+
reason?: string;
|
|
31
|
+
}
|
|
32
|
+
export type CancelTarget = CancelByRequestId | CancelByModelId;
|
|
33
|
+
/**
|
|
34
|
+
* Per-kind admission rule. Kinds without a registered policy have no
|
|
35
|
+
* admission control (every `begin(...)` is accepted as long as the
|
|
36
|
+
* request id is unique).
|
|
37
|
+
*
|
|
38
|
+
* The policy turns admission into a per-`(kind, modelId)` FIFO queue with
|
|
39
|
+
* a configurable concurrency limit. A second concurrent request to the
|
|
40
|
+
* same `(kind, modelId)` *waits its turn* (default) rather than colliding
|
|
41
|
+
* on the single native llama.cpp context. The limit is forward-compatible
|
|
42
|
+
* with addon-side continuous batching: bump `maxConcurrentPerModel` to the
|
|
43
|
+
* addon's slot count to flip serial execution to N-way concurrent without
|
|
44
|
+
* any further change.
|
|
45
|
+
*
|
|
46
|
+
* Requests without a `modelId` are never gated (there is no per-model
|
|
47
|
+
* identity to serialize against).
|
|
48
|
+
*/
|
|
49
|
+
export interface ConcurrencyPolicy {
|
|
50
|
+
kind: RequestKind;
|
|
51
|
+
/**
|
|
52
|
+
* Max simultaneously in-flight requests per `(kind, modelId)`.
|
|
53
|
+
* Default: `Infinity` (no admission limit). Set `1` to serialize, or
|
|
54
|
+
* `N` for N-way concurrency (the future continuous-batching slot
|
|
55
|
+
* count). Non-finite values disable gating entirely.
|
|
56
|
+
*/
|
|
57
|
+
maxConcurrentPerModel?: number;
|
|
58
|
+
/**
|
|
59
|
+
* What a `begin(...)` does when the `(kind, modelId)` is at capacity:
|
|
60
|
+
* - `"queue"` (default): wait FIFO for a slot to free.
|
|
61
|
+
* - `"reject"`: throw `RequestRejectedByPolicyError` immediately
|
|
62
|
+
* (the legacy `oneAtATimePerModel` behavior).
|
|
63
|
+
*/
|
|
64
|
+
onOverflow?: 'queue' | 'reject';
|
|
65
|
+
/**
|
|
66
|
+
* Max waiters allowed to queue per `(kind, modelId)` before further
|
|
67
|
+
* begins reject with `RequestRejectedByPolicyError`. Bounds memory so a
|
|
68
|
+
* runaway client can't grow the queue without limit. Default: `64`.
|
|
69
|
+
* Only consulted when `onOverflow` is `"queue"`.
|
|
70
|
+
*/
|
|
71
|
+
maxQueueDepthPerModel?: number;
|
|
72
|
+
/**
|
|
73
|
+
* Reject a waiter that has waited longer than this many milliseconds
|
|
74
|
+
* with `RequestRejectedByPolicyError`. Default: `undefined` (wait
|
|
75
|
+
* indefinitely for a slot).
|
|
76
|
+
*/
|
|
77
|
+
queueTimeoutMs?: number;
|
|
78
|
+
/**
|
|
79
|
+
* @deprecated Back-compat alias. `true` normalizes to
|
|
80
|
+
* `{ maxConcurrentPerModel: 1, onOverflow: "reject" }`; `false` (or
|
|
81
|
+
* omitted) leaves admission unlimited. Ignored when
|
|
82
|
+
* `maxConcurrentPerModel` is set explicitly.
|
|
83
|
+
*/
|
|
84
|
+
oneAtATimePerModel?: boolean;
|
|
85
|
+
/**
|
|
86
|
+
* Opt several kinds into one shared admission lane per model. When set,
|
|
87
|
+
* the slot is keyed on `(sharedSlotGroup, modelId)` instead of
|
|
88
|
+
* `(kind, modelId)`, so requests of *different* kinds that target the
|
|
89
|
+
* same model contend for the same slot pool.
|
|
90
|
+
*
|
|
91
|
+
* This exists to mirror an addon that serializes unrelated request
|
|
92
|
+
* kinds on a single native context — e.g. `@qvac/llm-llamacpp` funnels
|
|
93
|
+
* single-prompt `completion` and `batchCompletion` through one
|
|
94
|
+
* per-instance run queue, so the two can't actually run at once on the
|
|
95
|
+
* same model. Sharing a lane makes the admission queue the
|
|
96
|
+
* authoritative serialization point rather than relying on the addon's
|
|
97
|
+
* internal queue.
|
|
98
|
+
*/
|
|
99
|
+
sharedSlotGroup?: string;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* `ManagedRequestContext` is the value `begin(...)` resolves to. It extends
|
|
103
|
+
* `RequestContext` with an async-dispose method so handlers can write:
|
|
104
|
+
*
|
|
105
|
+
* await using ctx = await registry.begin({ ... });
|
|
106
|
+
*
|
|
107
|
+
* On dispose the scope unwinds (LIFO cleanup) and the registry slot is
|
|
108
|
+
* freed. If the handler doesn't override `ctx.state` before unwinding,
|
|
109
|
+
* the registry derives the terminal state from `signal.aborted` —
|
|
110
|
+
* `"cancelled"` when an abort was recorded, `"completed"` otherwise.
|
|
111
|
+
*/
|
|
112
|
+
export interface ManagedRequestContext extends RequestContext {
|
|
113
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
114
|
+
}
|
|
115
|
+
export interface RequestRegistry {
|
|
116
|
+
/**
|
|
117
|
+
* Open a new request. Returns a promise because admission may *queue*:
|
|
118
|
+
* when a concurrency policy caps the `(kind, modelId)` and it's at
|
|
119
|
+
* capacity with `onOverflow: "queue"`, the promise resolves once a slot
|
|
120
|
+
* frees (FIFO). Callers write `await using ctx = await registry.begin(...)`.
|
|
121
|
+
*
|
|
122
|
+
* Rejects with:
|
|
123
|
+
* - `RequestIdConflictError` if `requestId` is already present
|
|
124
|
+
* (UUIDv4 collision is astronomically unlikely; the guard exists
|
|
125
|
+
* so a buggy client retry sending the same id can't silently
|
|
126
|
+
* overwrite an in-flight request).
|
|
127
|
+
* - `RequestRejectedByPolicyError` if a concurrency policy was
|
|
128
|
+
* registered for `opts.kind` and the new request can't be admitted:
|
|
129
|
+
* `onOverflow: "reject"` at capacity, the queue depth cap is
|
|
130
|
+
* exceeded, or the waiter's `queueTimeoutMs` elapsed.
|
|
131
|
+
*/
|
|
132
|
+
begin(opts: BeginOpts): Promise<ManagedRequestContext>;
|
|
133
|
+
/**
|
|
134
|
+
* Register or replace the concurrency policy for a `RequestKind`.
|
|
135
|
+
* Subsequent `begin(...)` calls for that kind run the policy before
|
|
136
|
+
* allocating a controller / scope. One policy per kind — calling
|
|
137
|
+
* twice replaces the previous declaration.
|
|
138
|
+
*
|
|
139
|
+
* @example
|
|
140
|
+
* r.policy({ kind: "completion", maxConcurrentPerModel: 1, onOverflow: "reject" });
|
|
141
|
+
* await using a = await r.begin({ requestId: "r-1", kind: "completion", modelId: "m1" });
|
|
142
|
+
* await r.begin({ requestId: "r-2", kind: "completion", modelId: "m1" });
|
|
143
|
+
* // → rejects with RequestRejectedByPolicyError (code 52420)
|
|
144
|
+
*/
|
|
145
|
+
policy(opts: ConcurrencyPolicy): void;
|
|
146
|
+
/** Look up an in-flight request by id. */
|
|
147
|
+
get(requestId: string): RequestContext | null;
|
|
148
|
+
/**
|
|
149
|
+
* Snapshot of currently-tracked requests. Useful for diagnostics /
|
|
150
|
+
* structured logs ("which requests are in flight right now?"). Returns
|
|
151
|
+
* a fresh array; mutations on it are not observed by the registry.
|
|
152
|
+
*/
|
|
153
|
+
list(): RequestContext[];
|
|
154
|
+
/**
|
|
155
|
+
* Cancel matching requests. Returns the number of contexts whose abort
|
|
156
|
+
* was triggered by *this* call (already-cancelled contexts are skipped
|
|
157
|
+
* so callers can rely on the count to log "n requests cancelled" once).
|
|
158
|
+
*
|
|
159
|
+
* For `{ modelId }` and an optional `kind`, cancels every active
|
|
160
|
+
* request that matches the predicate. This is the broad-cancel path
|
|
161
|
+
* the pre-registry `cancel({ modelId })` API maps to.
|
|
162
|
+
*/
|
|
163
|
+
cancel(target: CancelTarget): number;
|
|
164
|
+
/**
|
|
165
|
+
* Cancel every active request — the process-shutdown / model-unload
|
|
166
|
+
* sweep. The reason is forwarded to each request as the abort reason
|
|
167
|
+
* so handler logs can distinguish a normal cancel from a sweep.
|
|
168
|
+
* Resolves once all targeted contexts have flipped to `"cancelling"`;
|
|
169
|
+
* scope unwinding still happens on each handler's own dispose path.
|
|
170
|
+
*/
|
|
171
|
+
cancelAll(reason: 'shutdown' | 'modelUnload'): Promise<void>;
|
|
172
|
+
/**
|
|
173
|
+
* Mark a request finished and dispose its scope. Equivalent to
|
|
174
|
+
* `await ctx[Symbol.asyncDispose]()` with an explicit outcome.
|
|
175
|
+
* Idempotent — calling `end` after a scope dispose is a no-op.
|
|
176
|
+
*/
|
|
177
|
+
end(requestId: string, outcome: RequestOutcome): Promise<void>;
|
|
178
|
+
}
|
|
179
|
+
export declare function createRequestRegistry(options?: {
|
|
180
|
+
/** Defaults to `getEngineLogger()`. Tests inject a stub. */
|
|
181
|
+
logger?: Logger;
|
|
182
|
+
}): RequestRegistry;
|
|
183
|
+
/**
|
|
184
|
+
* Test-only knobs exported for `request-registry.test.ts` so the bound
|
|
185
|
+
* assertions can pin the documented limits without re-reading them via
|
|
186
|
+
* fragile string comparison. **Not part of the public surface.**
|
|
187
|
+
*
|
|
188
|
+
* @internal
|
|
189
|
+
*/
|
|
190
|
+
export declare const __requestRegistryTestHooks: {
|
|
191
|
+
cancelBeforeBeginMaxEntries: number;
|
|
192
|
+
cancelBeforeBeginTtlMs: number;
|
|
193
|
+
defaultMaxQueueDepthPerModel: number;
|
|
194
|
+
};
|