@qvac/inference 0.0.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +179 -0
- package/NOTICE +986 -0
- package/README.md +138 -0
- package/dist/api/audio-gen.d.ts +18 -0
- package/dist/api/audio-gen.js +148 -0
- package/dist/api/batch-completion.d.ts +14 -0
- package/dist/api/batch-completion.js +332 -0
- package/dist/api/bci-transcribe.d.ts +56 -0
- package/dist/api/bci-transcribe.js +172 -0
- package/dist/api/cancel.d.ts +34 -0
- package/dist/api/cancel.js +82 -0
- package/dist/api/classify.d.ts +22 -0
- package/dist/api/classify.js +46 -0
- package/dist/api/completion-stream.d.ts +106 -0
- package/dist/api/completion-stream.js +351 -0
- package/dist/api/delete-cache.d.ts +29 -0
- package/dist/api/delete-cache.js +49 -0
- package/dist/api/diffusion.d.ts +92 -0
- package/dist/api/diffusion.js +176 -0
- package/dist/api/download-asset.d.ts +49 -0
- package/dist/api/download-asset.js +87 -0
- package/dist/api/embed.d.ts +41 -0
- package/dist/api/embed.js +29 -0
- package/dist/api/finetune.d.ts +89 -0
- package/dist/api/finetune.js +138 -0
- package/dist/api/get-loaded-model-info.d.ts +24 -0
- package/dist/api/get-loaded-model-info.js +34 -0
- package/dist/api/get-model-info.d.ts +43 -0
- package/dist/api/get-model-info.js +21 -0
- package/dist/api/get-system-resources.d.ts +2 -0
- package/dist/api/get-system-resources.js +16 -0
- package/dist/api/heartbeat.d.ts +29 -0
- package/dist/api/heartbeat.js +37 -0
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +35 -0
- package/dist/api/invoke-plugin.d.ts +30 -0
- package/dist/api/invoke-plugin.js +54 -0
- package/dist/api/load-model.d.ts +169 -0
- package/dist/api/load-model.js +129 -0
- package/dist/api/logging-stream-registry.d.ts +4 -0
- package/dist/api/logging-stream-registry.js +61 -0
- package/dist/api/logging-stream.d.ts +23 -0
- package/dist/api/logging-stream.js +36 -0
- package/dist/api/ocr.d.ts +30 -0
- package/dist/api/ocr.js +88 -0
- package/dist/api/parse-input.d.ts +2 -0
- package/dist/api/parse-input.js +14 -0
- package/dist/api/provide.d.ts +23 -0
- package/dist/api/provide.js +32 -0
- package/dist/api/rag.d.ts +251 -0
- package/dist/api/rag.js +485 -0
- package/dist/api/registry.d.ts +39 -0
- package/dist/api/registry.js +63 -0
- package/dist/api/resume.d.ts +29 -0
- package/dist/api/resume.js +36 -0
- package/dist/api/state.d.ts +22 -0
- package/dist/api/state.js +29 -0
- package/dist/api/stop-provider.d.ts +20 -0
- package/dist/api/stop-provider.js +30 -0
- package/dist/api/subscribe-logs.d.ts +27 -0
- package/dist/api/subscribe-logs.js +43 -0
- package/dist/api/suspend.d.ts +30 -0
- package/dist/api/suspend.js +37 -0
- package/dist/api/text-to-speech.d.ts +64 -0
- package/dist/api/text-to-speech.js +434 -0
- package/dist/api/transcribe.d.ts +75 -0
- package/dist/api/transcribe.js +256 -0
- package/dist/api/translate.d.ts +47 -0
- package/dist/api/translate.js +115 -0
- package/dist/api/unload-model.d.ts +17 -0
- package/dist/api/unload-model.js +42 -0
- package/dist/api/upscale.d.ts +44 -0
- package/dist/api/upscale.js +99 -0
- package/dist/api/video.d.ts +86 -0
- package/dist/api/video.js +176 -0
- package/dist/api/vla-helpers.d.ts +43 -0
- package/dist/api/vla-helpers.js +124 -0
- package/dist/api/vla.d.ts +68 -0
- package/dist/api/vla.js +113 -0
- package/dist/config/config-utils.d.ts +4 -0
- package/dist/config/config-utils.js +18 -0
- package/dist/config/resolve-config.d.ts +8 -0
- package/dist/config/resolve-config.js +74 -0
- package/dist/constants/audio.d.ts +1 -0
- package/dist/constants/audio.js +1 -0
- package/dist/constants/index.d.ts +2 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/registry.d.ts +1 -0
- package/dist/constants/registry.js +1 -0
- package/dist/dispatch.d.ts +18 -0
- package/dist/dispatch.js +271 -0
- package/dist/engine.d.ts +5 -0
- package/dist/engine.js +27 -0
- package/dist/errors/index.d.ts +416 -0
- package/dist/errors/index.js +681 -0
- package/dist/errors/options.d.ts +2 -0
- package/dist/errors/options.js +13 -0
- package/dist/handlers/cancelHandler.d.ts +29 -0
- package/dist/handlers/cancelHandler.js +67 -0
- package/dist/handlers/completion-orchestrate.d.ts +41 -0
- package/dist/handlers/completion-orchestrate.js +190 -0
- package/dist/handlers/delete-cache.d.ts +11 -0
- package/dist/handlers/delete-cache.js +37 -0
- package/dist/handlers/download-asset.d.ts +2 -0
- package/dist/handlers/download-asset.js +146 -0
- package/dist/handlers/finetune.d.ts +2 -0
- package/dist/handlers/finetune.js +18 -0
- package/dist/handlers/get-loaded-model-info.d.ts +2 -0
- package/dist/handlers/get-loaded-model-info.js +45 -0
- package/dist/handlers/get-model-info.d.ts +2 -0
- package/dist/handlers/get-model-info.js +173 -0
- package/dist/handlers/get-system-resources.d.ts +2 -0
- package/dist/handlers/get-system-resources.js +53 -0
- package/dist/handlers/heartbeat.d.ts +2 -0
- package/dist/handlers/heartbeat.js +3 -0
- package/dist/handlers/load-model/download-manager.d.ts +95 -0
- package/dist/handlers/load-model/download-manager.js +306 -0
- package/dist/handlers/load-model/download-stats.d.ts +7 -0
- package/dist/handlers/load-model/download-stats.js +126 -0
- package/dist/handlers/load-model/handler.d.ts +2 -0
- package/dist/handlers/load-model/handler.js +287 -0
- package/dist/handlers/load-model/http.d.ts +3 -0
- package/dist/handlers/load-model/http.js +857 -0
- package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
- package/dist/handlers/load-model/hyperdrive.js +560 -0
- package/dist/handlers/load-model/index.d.ts +2 -0
- package/dist/handlers/load-model/index.js +2 -0
- package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/handlers/load-model/registry-client-options.js +18 -0
- package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/handlers/load-model/registry-companion-set.js +206 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
- package/dist/handlers/load-model/registry-download-utils.js +112 -0
- package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
- package/dist/handlers/load-model/registry-metadata.js +23 -0
- package/dist/handlers/load-model/registry.d.ts +4 -0
- package/dist/handlers/load-model/registry.js +298 -0
- package/dist/handlers/load-model/resolve-session.d.ts +33 -0
- package/dist/handlers/load-model/resolve-session.js +62 -0
- package/dist/handlers/load-model/resolve.d.ts +5 -0
- package/dist/handlers/load-model/resolve.js +158 -0
- package/dist/handlers/load-model/types.d.ts +59 -0
- package/dist/handlers/load-model/types.js +28 -0
- package/dist/handlers/logging-stream.d.ts +2 -0
- package/dist/handlers/logging-stream.js +39 -0
- package/dist/handlers/plugin-dispatch.d.ts +3 -0
- package/dist/handlers/plugin-dispatch.js +63 -0
- package/dist/handlers/plugin-invoke.d.ts +3 -0
- package/dist/handlers/plugin-invoke.js +81 -0
- package/dist/handlers/registry.d.ts +4 -0
- package/dist/handlers/registry.js +156 -0
- package/dist/handlers/resume.d.ts +2 -0
- package/dist/handlers/resume.js +11 -0
- package/dist/handlers/state.d.ts +2 -0
- package/dist/handlers/state.js +7 -0
- package/dist/handlers/suspend.d.ts +2 -0
- package/dist/handlers/suspend.js +11 -0
- package/dist/handlers/types.d.ts +13 -0
- package/dist/handlers/types.js +1 -0
- package/dist/handlers/unload-model.d.ts +2 -0
- package/dist/handlers/unload-model.js +29 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +12 -0
- package/dist/logging/addon.d.ts +5 -0
- package/dist/logging/addon.js +75 -0
- package/dist/logging/index.d.ts +5 -0
- package/dist/logging/index.js +4 -0
- package/dist/logging/logger.d.ts +41 -0
- package/dist/logging/logger.js +209 -0
- package/dist/logging/namespaces.d.ts +6 -0
- package/dist/logging/namespaces.js +7 -0
- package/dist/logging/types.d.ts +37 -0
- package/dist/logging/types.js +1 -0
- package/dist/logging/utils.d.ts +5 -0
- package/dist/logging/utils.js +113 -0
- package/dist/models/registry/index.d.ts +18 -0
- package/dist/models/registry/index.js +21 -0
- package/dist/models/registry/models.d.ts +22680 -0
- package/dist/models/registry/models.js +23908 -0
- package/dist/p2p/cancel-delegated.d.ts +3 -0
- package/dist/p2p/cancel-delegated.js +64 -0
- package/dist/p2p/completion-stream-delegated.d.ts +4 -0
- package/dist/p2p/completion-stream-delegated.js +51 -0
- package/dist/p2p/delegate-client.d.ts +9 -0
- package/dist/p2p/delegate-client.js +219 -0
- package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
- package/dist/p2p/delegate-connect-diagnostics.js +38 -0
- package/dist/p2p/delegate-transport.d.ts +18 -0
- package/dist/p2p/delegate-transport.js +237 -0
- package/dist/p2p/error-handlers.d.ts +4 -0
- package/dist/p2p/error-handlers.js +31 -0
- package/dist/p2p/handle-request.d.ts +2 -0
- package/dist/p2p/handle-request.js +171 -0
- package/dist/p2p/handler-utils.d.ts +19 -0
- package/dist/p2p/handler-utils.js +180 -0
- package/dist/p2p/heartbeat-delegated.d.ts +3 -0
- package/dist/p2p/heartbeat-delegated.js +32 -0
- package/dist/p2p/load-model-delegated.d.ts +6 -0
- package/dist/p2p/load-model-delegated.js +132 -0
- package/dist/p2p/progress-throttle.d.ts +7 -0
- package/dist/p2p/progress-throttle.js +47 -0
- package/dist/p2p/provideHandler/connection.d.ts +2 -0
- package/dist/p2p/provideHandler/connection.js +38 -0
- package/dist/p2p/provideHandler/handler.d.ts +2 -0
- package/dist/p2p/provideHandler/handler.js +66 -0
- package/dist/p2p/provideHandler/proxy.d.ts +2 -0
- package/dist/p2p/provideHandler/proxy.js +24 -0
- package/dist/p2p/provideHandler.d.ts +1 -0
- package/dist/p2p/provideHandler.js +2 -0
- package/dist/p2p/rpc-utils.d.ts +4 -0
- package/dist/p2p/rpc-utils.js +8 -0
- package/dist/p2p/stopProvideHandler.d.ts +2 -0
- package/dist/p2p/stopProvideHandler.js +29 -0
- package/dist/p2p/swarm.d.ts +9 -0
- package/dist/p2p/swarm.js +103 -0
- package/dist/p2p/transport-selector.d.ts +2 -0
- package/dist/p2p/transport-selector.js +18 -0
- package/dist/p2p/unload-model-delegated.d.ts +2 -0
- package/dist/p2p/unload-model-delegated.js +31 -0
- package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
- package/dist/plugins/builtin/asr-ggml/config.js +46 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
- package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
- package/dist/plugins/builtin/index.d.ts +11 -0
- package/dist/plugins/builtin/index.js +11 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
- package/dist/plugins/index.d.ts +9 -0
- package/dist/plugins/index.js +12 -0
- package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
- package/dist/plugins/ops/bci-transcribe.js +248 -0
- package/dist/plugins/ops/cancel.d.ts +25 -0
- package/dist/plugins/ops/cancel.js +43 -0
- package/dist/plugins/ops/embed.d.ts +15 -0
- package/dist/plugins/ops/embed.js +193 -0
- package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
- package/dist/plugins/ops/kv-cache-retention.js +121 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
- package/dist/plugins/ops/kv-cache-utils.js +120 -0
- package/dist/plugins/ops/load-model.d.ts +8 -0
- package/dist/plugins/ops/load-model.js +82 -0
- package/dist/plugins/ops/transcribe.d.ts +20 -0
- package/dist/plugins/ops/transcribe.js +343 -0
- package/dist/plugins/ops/translate.d.ts +7 -0
- package/dist/plugins/ops/translate.js +256 -0
- package/dist/plugins/ops/unload-model.d.ts +2 -0
- package/dist/plugins/ops/unload-model.js +45 -0
- package/dist/plugins/registry.d.ts +9 -0
- package/dist/plugins/registry.js +107 -0
- package/dist/profiling/aggregator.d.ts +14 -0
- package/dist/profiling/aggregator.js +100 -0
- package/dist/profiling/backend-diagnostics.d.ts +8 -0
- package/dist/profiling/backend-diagnostics.js +36 -0
- package/dist/profiling/clock.d.ts +8 -0
- package/dist/profiling/clock.js +23 -0
- package/dist/profiling/controller.d.ts +34 -0
- package/dist/profiling/controller.js +130 -0
- package/dist/profiling/delegation-profiler.d.ts +31 -0
- package/dist/profiling/delegation-profiler.js +155 -0
- package/dist/profiling/envelope.d.ts +10 -0
- package/dist/profiling/envelope.js +45 -0
- package/dist/profiling/events.d.ts +19 -0
- package/dist/profiling/events.js +35 -0
- package/dist/profiling/exporters.d.ts +9 -0
- package/dist/profiling/exporters.js +242 -0
- package/dist/profiling/index.d.ts +91 -0
- package/dist/profiling/index.js +87 -0
- package/dist/profiling/model-execution.d.ts +15 -0
- package/dist/profiling/model-execution.js +34 -0
- package/dist/profiling/operation-metrics.d.ts +24 -0
- package/dist/profiling/operation-metrics.js +370 -0
- package/dist/profiling/operation-wrappers.d.ts +17 -0
- package/dist/profiling/operation-wrappers.js +196 -0
- package/dist/profiling/profiler.d.ts +27 -0
- package/dist/profiling/profiler.js +127 -0
- package/dist/profiling/ring-buffer.d.ts +18 -0
- package/dist/profiling/ring-buffer.js +83 -0
- package/dist/profiling/types.d.ts +52 -0
- package/dist/profiling/types.js +1 -0
- package/dist/rag/handler.d.ts +2 -0
- package/dist/rag/handler.js +272 -0
- package/dist/rag/handlers/chunk.d.ts +2 -0
- package/dist/rag/handlers/chunk.js +7 -0
- package/dist/rag/handlers/close-workspace.d.ts +2 -0
- package/dist/rag/handlers/close-workspace.js +9 -0
- package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
- package/dist/rag/handlers/delete-embeddings.js +11 -0
- package/dist/rag/handlers/delete-workspace.d.ts +2 -0
- package/dist/rag/handlers/delete-workspace.js +14 -0
- package/dist/rag/handlers/ingest.d.ts +9 -0
- package/dist/rag/handlers/ingest.js +19 -0
- package/dist/rag/handlers/list-workspaces.d.ts +2 -0
- package/dist/rag/handlers/list-workspaces.js +4 -0
- package/dist/rag/handlers/reindex.d.ts +9 -0
- package/dist/rag/handlers/reindex.js +16 -0
- package/dist/rag/handlers/save-embeddings.d.ts +9 -0
- package/dist/rag/handlers/save-embeddings.js +15 -0
- package/dist/rag/handlers/search.d.ts +2 -0
- package/dist/rag/handlers/search.js +16 -0
- package/dist/rag/index.d.ts +11 -0
- package/dist/rag/index.js +11 -0
- package/dist/rag/rag-operation-manager.d.ts +28 -0
- package/dist/rag/rag-operation-manager.js +76 -0
- package/dist/rag/rag-workspace-manager.d.ts +14 -0
- package/dist/rag/rag-workspace-manager.js +143 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +166 -0
- package/dist/resources/collector.d.ts +643 -0
- package/dist/resources/collector.js +226 -0
- package/dist/resources/instance.d.ts +4 -0
- package/dist/resources/instance.js +15 -0
- package/dist/resources/native.d.ts +2 -0
- package/dist/resources/native.js +35 -0
- package/dist/resources/normalize.d.ts +12 -0
- package/dist/resources/normalize.js +73 -0
- package/dist/resources/types.d.ts +65 -0
- package/dist/resources/types.js +1 -0
- package/dist/runtime/cache-lock.d.ts +2 -0
- package/dist/runtime/cache-lock.js +67 -0
- package/dist/runtime/disposable-scope.d.ts +35 -0
- package/dist/runtime/disposable-scope.js +69 -0
- package/dist/runtime/env.d.ts +16 -0
- package/dist/runtime/env.js +53 -0
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +20 -0
- package/dist/runtime/lifecycle.js +102 -0
- package/dist/runtime/logging-stream-registry.d.ts +27 -0
- package/dist/runtime/logging-stream-registry.js +133 -0
- package/dist/runtime/model-config-utils.d.ts +14 -0
- package/dist/runtime/model-config-utils.js +128 -0
- package/dist/runtime/model-registry.d.ts +74 -0
- package/dist/runtime/model-registry.js +130 -0
- package/dist/runtime/registry-client.d.ts +4 -0
- package/dist/runtime/registry-client.js +118 -0
- package/dist/runtime/request-context.d.ts +72 -0
- package/dist/runtime/request-context.js +114 -0
- package/dist/runtime/request-id.d.ts +17 -0
- package/dist/runtime/request-id.js +29 -0
- package/dist/runtime/request-registry.d.ts +194 -0
- package/dist/runtime/request-registry.js +546 -0
- package/dist/runtime/runtime-lifecycle.d.ts +37 -0
- package/dist/runtime/runtime-lifecycle.js +163 -0
- package/dist/runtime/state.d.ts +26 -0
- package/dist/runtime/state.js +138 -0
- package/dist/schemas/audio-gen.d.ts +252 -0
- package/dist/schemas/audio-gen.js +69 -0
- package/dist/schemas/batch-completion-stream.d.ts +467 -0
- package/dist/schemas/batch-completion-stream.js +123 -0
- package/dist/schemas/bci-config.d.ts +74 -0
- package/dist/schemas/bci-config.js +58 -0
- package/dist/schemas/bci.d.ts +178 -0
- package/dist/schemas/bci.js +89 -0
- package/dist/schemas/cancel.d.ts +203 -0
- package/dist/schemas/cancel.js +135 -0
- package/dist/schemas/classification.d.ts +52 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +6420 -0
- package/dist/schemas/common.js +133 -0
- package/dist/schemas/completion-event.d.ts +248 -0
- package/dist/schemas/completion-event.js +94 -0
- package/dist/schemas/completion-stream.d.ts +475 -0
- package/dist/schemas/completion-stream.js +236 -0
- package/dist/schemas/config.d.ts +5022 -0
- package/dist/schemas/config.js +169 -0
- package/dist/schemas/delegate.d.ts +29 -0
- package/dist/schemas/delegate.js +40 -0
- package/dist/schemas/download-asset.d.ts +225 -0
- package/dist/schemas/download-asset.js +71 -0
- package/dist/schemas/embed.d.ts +41 -0
- package/dist/schemas/embed.js +35 -0
- package/dist/schemas/engine-addon-map.d.ts +20 -0
- package/dist/schemas/engine-addon-map.js +78 -0
- package/dist/schemas/error.d.ts +40 -0
- package/dist/schemas/error.js +65 -0
- package/dist/schemas/errors.d.ts +119 -0
- package/dist/schemas/errors.js +615 -0
- package/dist/schemas/finetune.d.ts +621 -0
- package/dist/schemas/finetune.js +163 -0
- package/dist/schemas/index.d.ts +45 -0
- package/dist/schemas/index.js +45 -0
- package/dist/schemas/json-schema.d.ts +15 -0
- package/dist/schemas/json-schema.js +2 -0
- package/dist/schemas/lifecycle.d.ts +38 -0
- package/dist/schemas/lifecycle.js +21 -0
- package/dist/schemas/llamacpp-config.d.ts +403 -0
- package/dist/schemas/llamacpp-config.js +133 -0
- package/dist/schemas/load-model.d.ts +25635 -0
- package/dist/schemas/load-model.js +629 -0
- package/dist/schemas/logging-stream.d.ts +33 -0
- package/dist/schemas/logging-stream.js +17 -0
- package/dist/schemas/mcp-adapter.d.ts +46 -0
- package/dist/schemas/mcp-adapter.js +1 -0
- package/dist/schemas/model-file.d.ts +27 -0
- package/dist/schemas/model-file.js +29 -0
- package/dist/schemas/model-info.d.ts +197 -0
- package/dist/schemas/model-info.js +144 -0
- package/dist/schemas/model-ops.d.ts +36 -0
- package/dist/schemas/model-ops.js +40 -0
- package/dist/schemas/model-src-utils.d.ts +192 -0
- package/dist/schemas/model-src-utils.js +61 -0
- package/dist/schemas/model-types.d.ts +275 -0
- package/dist/schemas/model-types.js +223 -0
- package/dist/schemas/ocr.d.ts +138 -0
- package/dist/schemas/ocr.js +59 -0
- package/dist/schemas/path-validation.d.ts +9 -0
- package/dist/schemas/path-validation.js +22 -0
- package/dist/schemas/plugin.d.ts +341 -0
- package/dist/schemas/plugin.js +241 -0
- package/dist/schemas/profiling.d.ts +695 -0
- package/dist/schemas/profiling.js +100 -0
- package/dist/schemas/provide.d.ts +50 -0
- package/dist/schemas/provide.js +32 -0
- package/dist/schemas/rag.d.ts +301 -0
- package/dist/schemas/rag.js +211 -0
- package/dist/schemas/registry.d.ts +453 -0
- package/dist/schemas/registry.js +104 -0
- package/dist/schemas/reload-config.d.ts +585 -0
- package/dist/schemas/reload-config.js +42 -0
- package/dist/schemas/runtime-context.d.ts +18 -0
- package/dist/schemas/runtime-context.js +7 -0
- package/dist/schemas/sdcpp-config.d.ts +1126 -0
- package/dist/schemas/sdcpp-config.js +748 -0
- package/dist/schemas/system-resources.d.ts +3966 -0
- package/dist/schemas/system-resources.js +149 -0
- package/dist/schemas/text-to-speech.d.ts +1352 -0
- package/dist/schemas/text-to-speech.js +364 -0
- package/dist/schemas/tools.d.ts +80 -0
- package/dist/schemas/tools.js +52 -0
- package/dist/schemas/transcription-config.d.ts +161 -0
- package/dist/schemas/transcription-config.js +156 -0
- package/dist/schemas/transcription.d.ts +261 -0
- package/dist/schemas/transcription.js +145 -0
- package/dist/schemas/translate.d.ts +138 -0
- package/dist/schemas/translate.js +103 -0
- package/dist/schemas/translation-config.d.ts +1031 -0
- package/dist/schemas/translation-config.js +170 -0
- package/dist/schemas/vla.d.ts +110 -0
- package/dist/schemas/vla.js +151 -0
- package/dist/selection.d.ts +8 -0
- package/dist/selection.js +14 -0
- package/dist/surface.d.ts +18 -0
- package/dist/surface.js +30 -0
- package/dist/utils/addon-responses.d.ts +58 -0
- package/dist/utils/addon-responses.js +1 -0
- package/dist/utils/aggregate-events.d.ts +25 -0
- package/dist/utils/aggregate-events.js +73 -0
- package/dist/utils/archive.d.ts +4 -0
- package/dist/utils/archive.js +180 -0
- package/dist/utils/asr-events.d.ts +14 -0
- package/dist/utils/asr-events.js +30 -0
- package/dist/utils/audio/decoder.d.ts +4 -0
- package/dist/utils/audio/decoder.js +80 -0
- package/dist/utils/audio-input.d.ts +9 -0
- package/dist/utils/audio-input.js +34 -0
- package/dist/utils/cache/auto-history.d.ts +3 -0
- package/dist/utils/cache/auto-history.js +26 -0
- package/dist/utils/cache/paths.d.ts +42 -0
- package/dist/utils/cache/paths.js +133 -0
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/cache/types.js +1 -0
- package/dist/utils/cache-normalize.d.ts +9 -0
- package/dist/utils/cache-normalize.js +20 -0
- package/dist/utils/checksum.d.ts +3 -0
- package/dist/utils/checksum.js +29 -0
- package/dist/utils/clear-storage.d.ts +4 -0
- package/dist/utils/clear-storage.js +14 -0
- package/dist/utils/completion-normalizer.d.ts +17 -0
- package/dist/utils/completion-normalizer.js +453 -0
- package/dist/utils/config-transform.d.ts +2 -0
- package/dist/utils/config-transform.js +21 -0
- package/dist/utils/decorate-promise.d.ts +33 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/encoding.d.ts +16 -0
- package/dist/utils/encoding.js +27 -0
- package/dist/utils/expand-gguf-shards.d.ts +2 -0
- package/dist/utils/expand-gguf-shards.js +22 -0
- package/dist/utils/formatting.d.ts +18 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
- package/dist/utils/gguf-tensor-extractor.js +311 -0
- package/dist/utils/global-singleton.d.ts +1 -0
- package/dist/utils/global-singleton.js +10 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/load-model-validation.d.ts +6 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/dist/utils/mcp-adapter.d.ts +11 -0
- package/dist/utils/mcp-adapter.js +96 -0
- package/dist/utils/multi-gpu-mobile.d.ts +4 -0
- package/dist/utils/multi-gpu-mobile.js +10 -0
- package/dist/utils/object.d.ts +1 -0
- package/dist/utils/object.js +3 -0
- package/dist/utils/path-security.d.ts +22 -0
- package/dist/utils/path-security.js +69 -0
- package/dist/utils/qvac-paths.d.ts +1 -0
- package/dist/utils/qvac-paths.js +5 -0
- package/dist/utils/response-format.d.ts +2 -0
- package/dist/utils/response-format.js +20 -0
- package/dist/utils/shard-utils.d.ts +47 -0
- package/dist/utils/shard-utils.js +123 -0
- package/dist/utils/tool-helpers.d.ts +24 -0
- package/dist/utils/tool-helpers.js +111 -0
- package/dist/utils/tool-integration.d.ts +23 -0
- package/dist/utils/tool-integration.js +29 -0
- package/dist/utils/tools/dialect.d.ts +7 -0
- package/dist/utils/tools/dialect.js +20 -0
- package/dist/utils/tools/index.d.ts +2 -0
- package/dist/utils/tools/index.js +2 -0
- package/dist/utils/tools/parser.d.ts +5 -0
- package/dist/utils/tools/parser.js +69 -0
- package/dist/utils/tools/parsers/dsml.d.ts +3 -0
- package/dist/utils/tools/parsers/dsml.js +177 -0
- package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
- package/dist/utils/tools/parsers/gemma4native.js +65 -0
- package/dist/utils/tools/parsers/harmony.d.ts +3 -0
- package/dist/utils/tools/parsers/harmony.js +57 -0
- package/dist/utils/tools/parsers/hermes.d.ts +3 -0
- package/dist/utils/tools/parsers/hermes.js +122 -0
- package/dist/utils/tools/parsers/json.d.ts +5 -0
- package/dist/utils/tools/parsers/json.js +115 -0
- package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
- package/dist/utils/tools/parsers/pythonic.js +267 -0
- package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
- package/dist/utils/tools/parsers/qwen35.js +125 -0
- package/dist/utils/tools/shared.d.ts +17 -0
- package/dist/utils/tools/shared.js +52 -0
- package/dist/utils/transcribe-metadata.d.ts +20 -0
- package/dist/utils/transcribe-metadata.js +28 -0
- package/dist/utils/tts-stats.d.ts +25 -0
- package/dist/utils/tts-stats.js +16 -0
- package/dist/utils/withTimeout.d.ts +17 -0
- package/dist/utils/withTimeout.js +47 -0
- package/dist/utils/zod-error.d.ts +2 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +266 -1
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
import { AbortController } from 'bare-abort-controller';
|
|
2
|
+
import { createDisposableScope } from '../runtime/disposable-scope.js';
|
|
3
|
+
import { RequestIdConflictError, RequestRejectedByPolicyError } from '../errors/index.js';
|
|
4
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
5
|
+
/**
|
|
6
|
+
* Tuning knobs for the "cancelled-before-begin" bookkeeping set.
|
|
7
|
+
*
|
|
8
|
+
* The race window spans the gap between a `cancel({ requestId })` and
|
|
9
|
+
* the matching operation's `begin(...)`: a `cancel` issued at the same
|
|
10
|
+
* time as the matching `completion(...)` either lands first (and we need
|
|
11
|
+
* to remember it long enough for `begin(...)` to follow) or lands second
|
|
12
|
+
* (and we never touch this set). For a local call the gap is tiny; a
|
|
13
|
+
* delegated call adds the provider round-trip. 30 seconds is overkill for
|
|
14
|
+
* that but gives slow networks / pause-the-debugger scenarios enough
|
|
15
|
+
* slack while still bounding worst-case retention.
|
|
16
|
+
*
|
|
17
|
+
* The size cap protects against a buggy or malicious caller firing a
|
|
18
|
+
* stream of cancels for ids that never get a `begin(...)` follow-up —
|
|
19
|
+
* each `cancel({ requestId })` that doesn't match an in-flight context
|
|
20
|
+
* inserts one entry, so without a cap the process would grow the map
|
|
21
|
+
* unbounded. At the cap, the oldest entry is evicted.
|
|
22
|
+
*
|
|
23
|
+
* Tweak with care: both bounds appear in the registry race test
|
|
24
|
+
* (`bounded cancel-before-begin set does not grow past its cap`).
|
|
25
|
+
*/
|
|
26
|
+
const CANCEL_BEFORE_BEGIN_MAX_ENTRIES = 128;
|
|
27
|
+
const CANCEL_BEFORE_BEGIN_TTL_MS = 30_000;
|
|
28
|
+
/**
|
|
29
|
+
* Default cap on waiters queued per `(kind, modelId)` when a policy uses
|
|
30
|
+
* `onOverflow: "queue"` without an explicit `maxQueueDepthPerModel`.
|
|
31
|
+
* Bounds memory so a misbehaving client firing a flood of same-model
|
|
32
|
+
* requests can't grow the queue without limit — the (depth + 1)th begin
|
|
33
|
+
* rejects with `RequestRejectedByPolicyError` instead of enqueuing.
|
|
34
|
+
*/
|
|
35
|
+
const DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL = 64;
|
|
36
|
+
export function createRequestRegistry(options) {
|
|
37
|
+
const entries = new Map();
|
|
38
|
+
const policies = new Map();
|
|
39
|
+
const logger = options?.logger ?? getEngineLogger();
|
|
40
|
+
/**
|
|
41
|
+
* Per-`(kind, modelId)` admission semaphores. Keyed by
|
|
42
|
+
* `slotKey(kind, modelId)`. Absent until the first gated `begin(...)`
|
|
43
|
+
* for that key; removed again the moment the key is idle.
|
|
44
|
+
*/
|
|
45
|
+
const keyStates = new Map();
|
|
46
|
+
/**
|
|
47
|
+
* Reverse index from a queued waiter's `requestId` to its slot so
|
|
48
|
+
* `cancel({ requestId })` can pull a still-queued request out of the
|
|
49
|
+
* FIFO in O(1) without scanning every `KeyState`.
|
|
50
|
+
*/
|
|
51
|
+
const waitersById = new Map();
|
|
52
|
+
function logLifecycle(event, ctx, durationMs) {
|
|
53
|
+
const modelId = ctx.modelId !== undefined ? ctx.modelId : '-';
|
|
54
|
+
const base = `[request-lifecycle] ${event} requestId=${ctx.requestId} kind=${ctx.kind} modelId=${modelId} state=${ctx.state}`;
|
|
55
|
+
const line = durationMs !== undefined ? `${base} durationMs=${durationMs}` : base;
|
|
56
|
+
// `failed` end emits at `warn` so log shippers can alert on
|
|
57
|
+
// `level>=warn` for this prefix without parsing `state=failed`
|
|
58
|
+
// out of the message body. Everything else stays at `info`.
|
|
59
|
+
if (event === 'end' && ctx.state === 'failed') {
|
|
60
|
+
logger.warn(line);
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
logger.info(line);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* "Cancelled-before-begin" tripwire. A
|
|
68
|
+
* `cancel({ requestId })` whose target isn't yet in `entries` records
|
|
69
|
+
* the id here; the subsequent `begin({ requestId: <same id> })` then
|
|
70
|
+
* aborts the new controller before returning. Map order is insertion
|
|
71
|
+
* order — the iterator's first key is the oldest entry, which makes
|
|
72
|
+
* the size-cap eviction free.
|
|
73
|
+
*
|
|
74
|
+
* Invariants:
|
|
75
|
+
* - Every read path (`begin`, `cancel`-by-id) calls
|
|
76
|
+
* `pruneCancelBeforeBeginExpired()` first so a 30s+ stale entry
|
|
77
|
+
* never decides a fresh `begin(...)`.
|
|
78
|
+
* - Insertion enforces the size cap by evicting the oldest entry
|
|
79
|
+
* when at capacity — a malicious client cannot grow this map
|
|
80
|
+
* unbounded.
|
|
81
|
+
* - On a successful `begin(...)` match, the entry is consumed
|
|
82
|
+
* (removed) so a second `begin(...)` with the same id (which
|
|
83
|
+
* would itself throw `RequestIdConflictError`) doesn't see a
|
|
84
|
+
* phantom pre-cancel.
|
|
85
|
+
*/
|
|
86
|
+
const cancelledBeforeBegin = new Map();
|
|
87
|
+
function pruneCancelBeforeBeginExpired(now = Date.now()) {
|
|
88
|
+
if (cancelledBeforeBegin.size === 0)
|
|
89
|
+
return;
|
|
90
|
+
const cutoff = now - CANCEL_BEFORE_BEGIN_TTL_MS;
|
|
91
|
+
for (const [id, entry] of cancelledBeforeBegin) {
|
|
92
|
+
if (entry.at > cutoff)
|
|
93
|
+
break; // Insertion order ⇒ rest are newer.
|
|
94
|
+
cancelledBeforeBegin.delete(id);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function recordCancelBeforeBegin(requestId, reason) {
|
|
98
|
+
const now = Date.now();
|
|
99
|
+
pruneCancelBeforeBeginExpired(now);
|
|
100
|
+
// Re-canceling an id that is already tracked refreshes its TTL but
|
|
101
|
+
// keeps the original reason — the first cancel "won" the race.
|
|
102
|
+
if (cancelledBeforeBegin.has(requestId)) {
|
|
103
|
+
const existing = cancelledBeforeBegin.get(requestId);
|
|
104
|
+
cancelledBeforeBegin.delete(requestId);
|
|
105
|
+
cancelledBeforeBegin.set(requestId, { ...existing, at: now });
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
if (cancelledBeforeBegin.size >= CANCEL_BEFORE_BEGIN_MAX_ENTRIES) {
|
|
109
|
+
const oldest = cancelledBeforeBegin.keys().next().value;
|
|
110
|
+
if (oldest !== undefined)
|
|
111
|
+
cancelledBeforeBegin.delete(oldest);
|
|
112
|
+
}
|
|
113
|
+
cancelledBeforeBegin.set(requestId, reason !== undefined ? { at: now, reason } : { at: now });
|
|
114
|
+
}
|
|
115
|
+
function consumeCancelBeforeBegin(requestId) {
|
|
116
|
+
pruneCancelBeforeBeginExpired();
|
|
117
|
+
const entry = cancelledBeforeBegin.get(requestId);
|
|
118
|
+
if (!entry)
|
|
119
|
+
return undefined;
|
|
120
|
+
cancelledBeforeBegin.delete(requestId);
|
|
121
|
+
return entry;
|
|
122
|
+
}
|
|
123
|
+
function slotKey(lane, modelId) {
|
|
124
|
+
// NUL separator can't appear in a lane (a closed-union kind or a
|
|
125
|
+
// controlled `sharedSlotGroup` label) or a modelId (a generated handle),
|
|
126
|
+
// so the join is unambiguous.
|
|
127
|
+
return `${lane}\u0000${modelId}`;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Pull a waiter out of its queue and tidy up: clear its timeout, drop
|
|
131
|
+
* the reverse index, and delete the `KeyState` if that left it idle.
|
|
132
|
+
* Does NOT settle the waiter's promise — the caller decides whether the
|
|
133
|
+
* removal resolves (granted / graceful cancel) or rejects (timeout /
|
|
134
|
+
* teardown).
|
|
135
|
+
*/
|
|
136
|
+
function removeWaiter(key, waiter) {
|
|
137
|
+
if (waiter.timer !== undefined) {
|
|
138
|
+
clearTimeout(waiter.timer);
|
|
139
|
+
waiter.timer = undefined;
|
|
140
|
+
}
|
|
141
|
+
waitersById.delete(waiter.requestId);
|
|
142
|
+
const st = keyStates.get(key);
|
|
143
|
+
if (!st)
|
|
144
|
+
return;
|
|
145
|
+
const i = st.waiters.indexOf(waiter);
|
|
146
|
+
if (i >= 0)
|
|
147
|
+
st.waiters.splice(i, 1);
|
|
148
|
+
if (st.active <= 0 && st.waiters.length === 0)
|
|
149
|
+
keyStates.delete(key);
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Admission gate. Resolves to the slot key the caller must release on
|
|
153
|
+
* dispose, or `undefined` when the request is ungated (no policy, no
|
|
154
|
+
* `modelId`, an unbounded limit, or an already-aborted parent — which we
|
|
155
|
+
* let through so the normal abort path produces a clean cancelled
|
|
156
|
+
* outcome rather than queuing a doomed request).
|
|
157
|
+
*
|
|
158
|
+
* When the key is at capacity it either rejects (`onOverflow: "reject"`,
|
|
159
|
+
* queue-depth cap, or `queueTimeoutMs`) or enqueues a FIFO waiter and
|
|
160
|
+
* awaits it. The releaser hands a freed slot to the oldest waiter
|
|
161
|
+
* (`resolve(true)`), so `active` is never decremented and re-incremented
|
|
162
|
+
* across a hand-off — that's what guarantees FIFO fairness with no
|
|
163
|
+
* acquire/release race in the single-threaded event loop.
|
|
164
|
+
*/
|
|
165
|
+
async function acquireSlot(opts) {
|
|
166
|
+
if (opts.modelId === undefined)
|
|
167
|
+
return { slotKey: undefined };
|
|
168
|
+
const policy = policies.get(opts.kind);
|
|
169
|
+
if (!policy || !Number.isFinite(policy.maxConcurrent)) {
|
|
170
|
+
return { slotKey: undefined };
|
|
171
|
+
}
|
|
172
|
+
// A parent (process-shutdown) signal that's already aborted: don't
|
|
173
|
+
// queue behind live work that may never drain — let begin() proceed
|
|
174
|
+
// and abort immediately via the parentSignal path.
|
|
175
|
+
if (opts.parentSignal?.aborted)
|
|
176
|
+
return { slotKey: undefined };
|
|
177
|
+
const modelId = opts.modelId;
|
|
178
|
+
// A shared lane lets unrelated kinds (e.g. completion + batchCompletion)
|
|
179
|
+
// contend for one slot pool per model, matching an addon that serializes
|
|
180
|
+
// them on a single native context.
|
|
181
|
+
const key = slotKey(policy.slotGroup ?? opts.kind, modelId);
|
|
182
|
+
let st = keyStates.get(key);
|
|
183
|
+
if (!st) {
|
|
184
|
+
st = { active: 0, waiters: [] };
|
|
185
|
+
keyStates.set(key, st);
|
|
186
|
+
}
|
|
187
|
+
if (st.active < policy.maxConcurrent) {
|
|
188
|
+
st.active++;
|
|
189
|
+
return { slotKey: key };
|
|
190
|
+
}
|
|
191
|
+
if (policy.onOverflow === 'reject') {
|
|
192
|
+
throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `another ${opts.kind} request is already running on this model`);
|
|
193
|
+
}
|
|
194
|
+
if (st.waiters.length >= policy.maxQueueDepth) {
|
|
195
|
+
throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `request queue for this model is full (${st.waiters.length} waiting)`);
|
|
196
|
+
}
|
|
197
|
+
const queue = st;
|
|
198
|
+
const granted = await new Promise((resolve, reject) => {
|
|
199
|
+
const waiter = {
|
|
200
|
+
requestId: opts.requestId,
|
|
201
|
+
kind: opts.kind,
|
|
202
|
+
modelId,
|
|
203
|
+
enqueuedAt: Date.now(),
|
|
204
|
+
resolve,
|
|
205
|
+
reject,
|
|
206
|
+
timer: undefined
|
|
207
|
+
};
|
|
208
|
+
if (policy.queueTimeoutMs !== undefined) {
|
|
209
|
+
const timeoutMs = policy.queueTimeoutMs;
|
|
210
|
+
waiter.timer = setTimeout(() => {
|
|
211
|
+
removeWaiter(key, waiter);
|
|
212
|
+
reject(new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `timed out waiting ${timeoutMs}ms for a slot on this model`));
|
|
213
|
+
}, timeoutMs);
|
|
214
|
+
}
|
|
215
|
+
queue.waiters.push(waiter);
|
|
216
|
+
waitersById.set(opts.requestId, { key, waiter });
|
|
217
|
+
});
|
|
218
|
+
return granted ? { slotKey: key } : { slotKey: undefined };
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Hand a freed slot to the next FIFO waiter, or free it outright when
|
|
222
|
+
* none are queued. Called from `disposeEntry` after the scope has fully
|
|
223
|
+
* unwound, so the native context is genuinely free before the next
|
|
224
|
+
* same-model request is admitted.
|
|
225
|
+
*/
|
|
226
|
+
function releaseSlot(key) {
|
|
227
|
+
const st = keyStates.get(key);
|
|
228
|
+
if (!st)
|
|
229
|
+
return;
|
|
230
|
+
const next = st.waiters.shift();
|
|
231
|
+
if (next) {
|
|
232
|
+
// Hand off: the waiter inherits the slot, so `active` is unchanged.
|
|
233
|
+
waitersById.delete(next.requestId);
|
|
234
|
+
if (next.timer !== undefined) {
|
|
235
|
+
clearTimeout(next.timer);
|
|
236
|
+
next.timer = undefined;
|
|
237
|
+
}
|
|
238
|
+
next.resolve(true);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
st.active--;
|
|
242
|
+
if (st.active <= 0 && st.waiters.length === 0)
|
|
243
|
+
keyStates.delete(key);
|
|
244
|
+
}
|
|
245
|
+
function cancelEntry(entry, reason) {
|
|
246
|
+
if (entry.controller.signal.aborted)
|
|
247
|
+
return false;
|
|
248
|
+
entry.ctx.state = 'cancelling';
|
|
249
|
+
entry.controller.abort(reason);
|
|
250
|
+
logLifecycle('cancel', entry.ctx);
|
|
251
|
+
return true;
|
|
252
|
+
}
|
|
253
|
+
async function disposeEntry(entry, outcome) {
|
|
254
|
+
if (entry.scope.disposed)
|
|
255
|
+
return;
|
|
256
|
+
entry.ctx.state = outcome;
|
|
257
|
+
entry.detachParent();
|
|
258
|
+
logLifecycle('end', entry.ctx, Date.now() - entry.startedAt);
|
|
259
|
+
// Pull the entry out before unwinding so observers (e.g. a `cancel(...)`
|
|
260
|
+
// racing with dispose) don't see a half-disposed context.
|
|
261
|
+
entries.delete(entry.ctx.requestId);
|
|
262
|
+
try {
|
|
263
|
+
await entry.scope[Symbol.asyncDispose]();
|
|
264
|
+
}
|
|
265
|
+
finally {
|
|
266
|
+
// Release the admission slot only after the scope has fully unwound:
|
|
267
|
+
// the shared llama.cpp context / addon job is torn down in scope
|
|
268
|
+
// cleanup, so the next queued same-model request must not be admitted
|
|
269
|
+
// (and start decoding) until this one has actually let go. `finally`
|
|
270
|
+
// guarantees the slot is freed even if a deferred cleanup throws,
|
|
271
|
+
// otherwise a throwing teardown would strand the queue forever.
|
|
272
|
+
if (entry.slotKey !== undefined)
|
|
273
|
+
releaseSlot(entry.slotKey);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
async function begin(opts) {
|
|
277
|
+
// A request id is reserved for its whole lifecycle, including the time
|
|
278
|
+
// it spends *queued* for an admission slot. `waitersById` holds the
|
|
279
|
+
// still-queued begins (they aren't in `entries` yet), so a duplicate id
|
|
280
|
+
// must be rejected against both maps — otherwise a second begin with the
|
|
281
|
+
// same id would enqueue behind the first and overwrite its `waitersById`
|
|
282
|
+
// index, leaving the original waiter unreachable by `cancel({ requestId })`.
|
|
283
|
+
if (entries.has(opts.requestId) || waitersById.has(opts.requestId)) {
|
|
284
|
+
throw new RequestIdConflictError(opts.requestId);
|
|
285
|
+
}
|
|
286
|
+
// Admission control runs before allocation so a rejected begin leaves
|
|
287
|
+
// no controller / scope behind. This may await: a gated `(kind,
|
|
288
|
+
// modelId)` at capacity queues the begin FIFO and resolves once a slot
|
|
289
|
+
// frees (or rejects on overflow / queue-depth cap / timeout).
|
|
290
|
+
const { slotKey } = await acquireSlot(opts);
|
|
291
|
+
// The only interleaving point in this otherwise synchronous body is the
|
|
292
|
+
// await above. A duplicate id could (astronomically unlikely) have
|
|
293
|
+
// landed while we were queued — re-check and hand the just-acquired
|
|
294
|
+
// slot back rather than stranding it on the conflicting throw.
|
|
295
|
+
if (entries.has(opts.requestId)) {
|
|
296
|
+
if (slotKey !== undefined)
|
|
297
|
+
releaseSlot(slotKey);
|
|
298
|
+
throw new RequestIdConflictError(opts.requestId);
|
|
299
|
+
}
|
|
300
|
+
const controller = new AbortController();
|
|
301
|
+
const scope = createDisposableScope();
|
|
302
|
+
// Stop-button race close. If a
|
|
303
|
+
// `cancel({ requestId })` already arrived for this id, abort the
|
|
304
|
+
// new controller before observers can subscribe to it. The
|
|
305
|
+
// tripwire entry is consumed so a later, separate `begin(...)`
|
|
306
|
+
// with the same id is unaffected (in practice ids are UUIDv4 and
|
|
307
|
+
// never reused; this guard just keeps the contract self-
|
|
308
|
+
// consistent under retries).
|
|
309
|
+
const preCancel = consumeCancelBeforeBegin(opts.requestId);
|
|
310
|
+
if (preCancel) {
|
|
311
|
+
controller.abort(preCancel.reason);
|
|
312
|
+
}
|
|
313
|
+
let detachParent = () => { };
|
|
314
|
+
if (opts.parentSignal) {
|
|
315
|
+
const parent = opts.parentSignal;
|
|
316
|
+
if (parent.aborted) {
|
|
317
|
+
controller.abort(parent.reason);
|
|
318
|
+
}
|
|
319
|
+
else {
|
|
320
|
+
const onParentAbort = () => controller.abort(parent.reason);
|
|
321
|
+
parent.addEventListener('abort', onParentAbort, { once: true });
|
|
322
|
+
detachParent = () => parent.removeEventListener('abort', onParentAbort);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
const ctx = {
|
|
326
|
+
requestId: opts.requestId,
|
|
327
|
+
kind: opts.kind,
|
|
328
|
+
modelId: opts.modelId,
|
|
329
|
+
signal: controller.signal,
|
|
330
|
+
scope,
|
|
331
|
+
// Land the context in `cancelling` from the outset whenever the
|
|
332
|
+
// controller was already aborted by `begin(...)` itself — either
|
|
333
|
+
// the Stop-button race (`preCancel`) or a `parentSignal` that was
|
|
334
|
+
// already aborted at begin time. Both branches abort the
|
|
335
|
+
// controller above, so without this guard observers would see a
|
|
336
|
+
// momentarily-`running` context with an already-aborted signal.
|
|
337
|
+
state: preCancel || opts.parentSignal?.aborted ? 'cancelling' : 'running'
|
|
338
|
+
};
|
|
339
|
+
const entry = {
|
|
340
|
+
ctx,
|
|
341
|
+
controller,
|
|
342
|
+
scope,
|
|
343
|
+
detachParent,
|
|
344
|
+
startedAt: Date.now(),
|
|
345
|
+
slotKey
|
|
346
|
+
};
|
|
347
|
+
entries.set(opts.requestId, entry);
|
|
348
|
+
logLifecycle('begin', ctx);
|
|
349
|
+
return {
|
|
350
|
+
get requestId() {
|
|
351
|
+
return ctx.requestId;
|
|
352
|
+
},
|
|
353
|
+
get kind() {
|
|
354
|
+
return ctx.kind;
|
|
355
|
+
},
|
|
356
|
+
get modelId() {
|
|
357
|
+
return ctx.modelId;
|
|
358
|
+
},
|
|
359
|
+
get signal() {
|
|
360
|
+
return ctx.signal;
|
|
361
|
+
},
|
|
362
|
+
get scope() {
|
|
363
|
+
return ctx.scope;
|
|
364
|
+
},
|
|
365
|
+
get state() {
|
|
366
|
+
return ctx.state;
|
|
367
|
+
},
|
|
368
|
+
set state(next) {
|
|
369
|
+
ctx.state = next;
|
|
370
|
+
},
|
|
371
|
+
[Symbol.asyncDispose]: async () => {
|
|
372
|
+
await disposeEntry(entry, derivedTerminalState(ctx));
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
function get(requestId) {
|
|
377
|
+
return entries.get(requestId)?.ctx ?? null;
|
|
378
|
+
}
|
|
379
|
+
function list() {
|
|
380
|
+
return Array.from(entries.values(), (e) => e.ctx);
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* Cancel a still-queued waiter the same way the Stop-button race is
|
|
384
|
+
* handled: record a cancel-before-begin so its `begin(...)` — which
|
|
385
|
+
* resumes the instant we `resolve(false)` (without a slot) — observes the
|
|
386
|
+
* cancel and returns an already-aborted context. The request thus ends in
|
|
387
|
+
* a clean `cancelled` state rather than throwing.
|
|
388
|
+
*/
|
|
389
|
+
function cancelQueuedWaiterGracefully(key, waiter, reason) {
|
|
390
|
+
recordCancelBeforeBegin(waiter.requestId, reason);
|
|
391
|
+
removeWaiter(key, waiter);
|
|
392
|
+
waiter.resolve(false);
|
|
393
|
+
}
|
|
394
|
+
function cancel(target) {
|
|
395
|
+
let cancelled = 0;
|
|
396
|
+
if ('requestId' in target) {
|
|
397
|
+
const entry = entries.get(target.requestId);
|
|
398
|
+
if (entry) {
|
|
399
|
+
if (cancelEntry(entry, target.reason))
|
|
400
|
+
cancelled++;
|
|
401
|
+
return cancelled;
|
|
402
|
+
}
|
|
403
|
+
// Queued (still waiting for an admission slot)? It has no controller
|
|
404
|
+
// yet, so cancel it through the cancel-before-begin tripwire: pull it
|
|
405
|
+
// from the FIFO and let its begin() resume into an aborted context.
|
|
406
|
+
// Counted as one — unlike the pure race below, this targets a known
|
|
407
|
+
// in-flight (queued) begin, so the Stop button feels responsive
|
|
408
|
+
// instead of waiting for the slot to free first.
|
|
409
|
+
const queued = waitersById.get(target.requestId);
|
|
410
|
+
if (queued) {
|
|
411
|
+
cancelQueuedWaiterGracefully(queued.key, queued.waiter, target.reason);
|
|
412
|
+
return cancelled + 1;
|
|
413
|
+
}
|
|
414
|
+
// Stop-button race: the cancel beat its own
|
|
415
|
+
// `begin(...)`. Record the cancel so the next matching `begin`
|
|
416
|
+
// aborts immediately. The return value stays 0 — no in-flight
|
|
417
|
+
// request was matched, which is still the truth — but the
|
|
418
|
+
// *effective* cancel will land when the begin arrives.
|
|
419
|
+
recordCancelBeforeBegin(target.requestId, target.reason);
|
|
420
|
+
return cancelled;
|
|
421
|
+
}
|
|
422
|
+
for (const entry of entries.values()) {
|
|
423
|
+
if (entry.ctx.modelId !== target.modelId)
|
|
424
|
+
continue;
|
|
425
|
+
if (target.kind && entry.ctx.kind !== target.kind)
|
|
426
|
+
continue;
|
|
427
|
+
if (cancelEntry(entry, target.reason))
|
|
428
|
+
cancelled++;
|
|
429
|
+
}
|
|
430
|
+
// Queued waiters aren't in `entries`; drain the ones matching this
|
|
431
|
+
// model (and optional kind) so a broad `cancel({ modelId })` doesn't
|
|
432
|
+
// strand them behind a request that's being cancelled out from under
|
|
433
|
+
// them. Snapshot first — `removeWaiter` mutates `waitersById`.
|
|
434
|
+
for (const { key, waiter } of Array.from(waitersById.values())) {
|
|
435
|
+
if (waiter.modelId !== target.modelId)
|
|
436
|
+
continue;
|
|
437
|
+
if (target.kind && waiter.kind !== target.kind)
|
|
438
|
+
continue;
|
|
439
|
+
cancelQueuedWaiterGracefully(key, waiter, target.reason);
|
|
440
|
+
cancelled++;
|
|
441
|
+
}
|
|
442
|
+
return cancelled;
|
|
443
|
+
}
|
|
444
|
+
function cancelAll(reason) {
|
|
445
|
+
for (const entry of entries.values()) {
|
|
446
|
+
cancelEntry(entry, reason);
|
|
447
|
+
}
|
|
448
|
+
// Drain every queued waiter so a shutdown / model-unload sweep can't
|
|
449
|
+
// leave a `begin(...)` promise hung forever — which would in turn block
|
|
450
|
+
// the unload waiting on a request that never resolves. Unlike the
|
|
451
|
+
// targeted cancels above these *reject* rather than resolve-into-
|
|
452
|
+
// aborted: the model / process they queued against is being torn down,
|
|
453
|
+
// so there is nothing left for them to run. Snapshot first —
|
|
454
|
+
// `removeWaiter` mutates `waitersById`.
|
|
455
|
+
for (const { key, waiter } of Array.from(waitersById.values())) {
|
|
456
|
+
removeWaiter(key, waiter);
|
|
457
|
+
waiter.reject(new RequestRejectedByPolicyError(waiter.requestId, waiter.kind, waiter.modelId, `queued request cancelled (${reason})`));
|
|
458
|
+
}
|
|
459
|
+
// The interface returns Promise<void> so we can later make this an
|
|
460
|
+
// async sweep that awaits per-handler scope unwinding (e.g. join on
|
|
461
|
+
// the disposers). Today every handler unwinds on its own dispose
|
|
462
|
+
// path, so the function only needs to fire-and-forget the abort.
|
|
463
|
+
return Promise.resolve();
|
|
464
|
+
}
|
|
465
|
+
async function end(requestId, outcome) {
|
|
466
|
+
const entry = entries.get(requestId);
|
|
467
|
+
if (!entry)
|
|
468
|
+
return;
|
|
469
|
+
await disposeEntry(entry, outcome);
|
|
470
|
+
}
|
|
471
|
+
function policy(opts) {
|
|
472
|
+
policies.set(opts.kind, normalizePolicy(opts));
|
|
473
|
+
}
|
|
474
|
+
return {
|
|
475
|
+
begin,
|
|
476
|
+
get,
|
|
477
|
+
list,
|
|
478
|
+
cancel,
|
|
479
|
+
cancelAll,
|
|
480
|
+
end,
|
|
481
|
+
policy,
|
|
482
|
+
// Test-only: lets the registry race tests assert the bound
|
|
483
|
+
// invariants on the internal "cancelled-before-begin" set without
|
|
484
|
+
// exposing it as a public surface. Kept off the `RequestRegistry`
|
|
485
|
+
// interface (typed via the augmented return type below) so handler
|
|
486
|
+
// code can't depend on it accidentally.
|
|
487
|
+
__cancelBeforeBeginSize: () => cancelledBeforeBegin.size,
|
|
488
|
+
// Test-only: the number of live per-`(kind, modelId)` admission
|
|
489
|
+
// states. Lets the queue tests assert the map empties on drain /
|
|
490
|
+
// dispose (no `KeyState` leak). Also off the public interface.
|
|
491
|
+
__keyStateSize: () => keyStates.size
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
/**
|
|
495
|
+
* Resolve a `ConcurrencyPolicy` to concrete numbers, applying defaults and
|
|
496
|
+
* the deprecated `oneAtATimePerModel` alias. Done once at registration so
|
|
497
|
+
* the `begin(...)` hot path reads plain numbers.
|
|
498
|
+
*/
|
|
499
|
+
function normalizePolicy(opts) {
|
|
500
|
+
let maxConcurrent;
|
|
501
|
+
let onOverflow;
|
|
502
|
+
if (opts.maxConcurrentPerModel !== undefined) {
|
|
503
|
+
maxConcurrent = opts.maxConcurrentPerModel;
|
|
504
|
+
onOverflow = opts.onOverflow ?? 'queue';
|
|
505
|
+
}
|
|
506
|
+
else if (opts.oneAtATimePerModel === true) {
|
|
507
|
+
// Legacy alias: at most one in-flight per model, reject the rest.
|
|
508
|
+
maxConcurrent = 1;
|
|
509
|
+
onOverflow = opts.onOverflow ?? 'reject';
|
|
510
|
+
}
|
|
511
|
+
else {
|
|
512
|
+
// Nothing configured (includes `oneAtATimePerModel: false`) ⇒ no gating.
|
|
513
|
+
maxConcurrent = Infinity;
|
|
514
|
+
onOverflow = opts.onOverflow ?? 'queue';
|
|
515
|
+
}
|
|
516
|
+
// A finite limit below 1 would gate every request forever; clamp to the
|
|
517
|
+
// smallest sensible serial limit.
|
|
518
|
+
if (Number.isFinite(maxConcurrent) && maxConcurrent < 1)
|
|
519
|
+
maxConcurrent = 1;
|
|
520
|
+
return {
|
|
521
|
+
maxConcurrent,
|
|
522
|
+
onOverflow,
|
|
523
|
+
maxQueueDepth: opts.maxQueueDepthPerModel ?? DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL,
|
|
524
|
+
queueTimeoutMs: opts.queueTimeoutMs,
|
|
525
|
+
slotGroup: opts.sharedSlotGroup
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* Test-only knobs exported for `request-registry.test.ts` so the bound
|
|
530
|
+
* assertions can pin the documented limits without re-reading them via
|
|
531
|
+
* fragile string comparison. **Not part of the public surface.**
|
|
532
|
+
*
|
|
533
|
+
* @internal
|
|
534
|
+
*/
|
|
535
|
+
export const __requestRegistryTestHooks = {
|
|
536
|
+
cancelBeforeBeginMaxEntries: CANCEL_BEFORE_BEGIN_MAX_ENTRIES,
|
|
537
|
+
cancelBeforeBeginTtlMs: CANCEL_BEFORE_BEGIN_TTL_MS,
|
|
538
|
+
defaultMaxQueueDepthPerModel: DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL
|
|
539
|
+
};
|
|
540
|
+
function derivedTerminalState(ctx) {
|
|
541
|
+
if (ctx.state === 'failed')
|
|
542
|
+
return 'failed';
|
|
543
|
+
if (ctx.signal.aborted || ctx.state === 'cancelled')
|
|
544
|
+
return 'cancelled';
|
|
545
|
+
return 'completed';
|
|
546
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { LifecycleState, Request } from '../schemas/index.js';
|
|
2
|
+
export interface SuspendableSwarm {
|
|
3
|
+
readonly suspended: boolean;
|
|
4
|
+
suspend(): Promise<void>;
|
|
5
|
+
resume(): Promise<void>;
|
|
6
|
+
}
|
|
7
|
+
export interface SuspendableStore {
|
|
8
|
+
suspend(): Promise<void>;
|
|
9
|
+
resume(): Promise<void>;
|
|
10
|
+
}
|
|
11
|
+
interface ResourceMeta {
|
|
12
|
+
label: string;
|
|
13
|
+
createdAt: number;
|
|
14
|
+
}
|
|
15
|
+
type ResumeListener = () => void;
|
|
16
|
+
/**
|
|
17
|
+
* Register a callback fired after the runtime returns to "active". Lets in-flight
|
|
18
|
+
* work (e.g. an HTTP download whose socket died while backgrounded) recover the
|
|
19
|
+
* instant `resume()` runs instead of waiting for a timeout. Returns an
|
|
20
|
+
* unregister function — callers MUST call it when their work finishes so the
|
|
21
|
+
* listener doesn't leak into the next operation.
|
|
22
|
+
*/
|
|
23
|
+
export declare function onResume(listener: ResumeListener): () => void;
|
|
24
|
+
export declare function registerSwarm(swarm: SuspendableSwarm, meta: ResourceMeta): void;
|
|
25
|
+
export declare function unregisterSwarm(swarm: SuspendableSwarm): void;
|
|
26
|
+
export declare function registerCorestore(store: SuspendableStore, meta: ResourceMeta): void;
|
|
27
|
+
export declare function unregisterCorestore(store: SuspendableStore): void;
|
|
28
|
+
export declare function getLifecycleState(): LifecycleState;
|
|
29
|
+
export declare function assertLifecycleAllowed(request: Request): void;
|
|
30
|
+
export declare function getRegisteredResourceCounts(): {
|
|
31
|
+
swarms: number;
|
|
32
|
+
stores: number;
|
|
33
|
+
};
|
|
34
|
+
export declare function resetLifecycleState(): void;
|
|
35
|
+
export declare function suspendRuntime(): Promise<void>;
|
|
36
|
+
export declare function resumeRuntime(): Promise<void>;
|
|
37
|
+
export {};
|