@qvac/inference 0.0.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +179 -0
- package/NOTICE +986 -0
- package/README.md +138 -0
- package/dist/api/audio-gen.d.ts +18 -0
- package/dist/api/audio-gen.js +148 -0
- package/dist/api/batch-completion.d.ts +14 -0
- package/dist/api/batch-completion.js +332 -0
- package/dist/api/bci-transcribe.d.ts +56 -0
- package/dist/api/bci-transcribe.js +172 -0
- package/dist/api/cancel.d.ts +34 -0
- package/dist/api/cancel.js +82 -0
- package/dist/api/classify.d.ts +22 -0
- package/dist/api/classify.js +46 -0
- package/dist/api/completion-stream.d.ts +106 -0
- package/dist/api/completion-stream.js +351 -0
- package/dist/api/delete-cache.d.ts +29 -0
- package/dist/api/delete-cache.js +49 -0
- package/dist/api/diffusion.d.ts +92 -0
- package/dist/api/diffusion.js +176 -0
- package/dist/api/download-asset.d.ts +49 -0
- package/dist/api/download-asset.js +87 -0
- package/dist/api/embed.d.ts +41 -0
- package/dist/api/embed.js +29 -0
- package/dist/api/finetune.d.ts +89 -0
- package/dist/api/finetune.js +138 -0
- package/dist/api/get-loaded-model-info.d.ts +24 -0
- package/dist/api/get-loaded-model-info.js +34 -0
- package/dist/api/get-model-info.d.ts +43 -0
- package/dist/api/get-model-info.js +21 -0
- package/dist/api/get-system-resources.d.ts +2 -0
- package/dist/api/get-system-resources.js +16 -0
- package/dist/api/heartbeat.d.ts +29 -0
- package/dist/api/heartbeat.js +37 -0
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +35 -0
- package/dist/api/invoke-plugin.d.ts +30 -0
- package/dist/api/invoke-plugin.js +54 -0
- package/dist/api/load-model.d.ts +169 -0
- package/dist/api/load-model.js +129 -0
- package/dist/api/logging-stream-registry.d.ts +4 -0
- package/dist/api/logging-stream-registry.js +61 -0
- package/dist/api/logging-stream.d.ts +23 -0
- package/dist/api/logging-stream.js +36 -0
- package/dist/api/ocr.d.ts +30 -0
- package/dist/api/ocr.js +88 -0
- package/dist/api/parse-input.d.ts +2 -0
- package/dist/api/parse-input.js +14 -0
- package/dist/api/provide.d.ts +23 -0
- package/dist/api/provide.js +32 -0
- package/dist/api/rag.d.ts +251 -0
- package/dist/api/rag.js +485 -0
- package/dist/api/registry.d.ts +39 -0
- package/dist/api/registry.js +63 -0
- package/dist/api/resume.d.ts +29 -0
- package/dist/api/resume.js +36 -0
- package/dist/api/state.d.ts +22 -0
- package/dist/api/state.js +29 -0
- package/dist/api/stop-provider.d.ts +20 -0
- package/dist/api/stop-provider.js +30 -0
- package/dist/api/subscribe-logs.d.ts +27 -0
- package/dist/api/subscribe-logs.js +43 -0
- package/dist/api/suspend.d.ts +30 -0
- package/dist/api/suspend.js +37 -0
- package/dist/api/text-to-speech.d.ts +64 -0
- package/dist/api/text-to-speech.js +434 -0
- package/dist/api/transcribe.d.ts +75 -0
- package/dist/api/transcribe.js +256 -0
- package/dist/api/translate.d.ts +47 -0
- package/dist/api/translate.js +115 -0
- package/dist/api/unload-model.d.ts +17 -0
- package/dist/api/unload-model.js +42 -0
- package/dist/api/upscale.d.ts +44 -0
- package/dist/api/upscale.js +99 -0
- package/dist/api/video.d.ts +86 -0
- package/dist/api/video.js +176 -0
- package/dist/api/vla-helpers.d.ts +43 -0
- package/dist/api/vla-helpers.js +124 -0
- package/dist/api/vla.d.ts +68 -0
- package/dist/api/vla.js +113 -0
- package/dist/config/config-utils.d.ts +4 -0
- package/dist/config/config-utils.js +18 -0
- package/dist/config/resolve-config.d.ts +8 -0
- package/dist/config/resolve-config.js +74 -0
- package/dist/constants/audio.d.ts +1 -0
- package/dist/constants/audio.js +1 -0
- package/dist/constants/index.d.ts +2 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/registry.d.ts +1 -0
- package/dist/constants/registry.js +1 -0
- package/dist/dispatch.d.ts +18 -0
- package/dist/dispatch.js +271 -0
- package/dist/engine.d.ts +5 -0
- package/dist/engine.js +27 -0
- package/dist/errors/index.d.ts +416 -0
- package/dist/errors/index.js +681 -0
- package/dist/errors/options.d.ts +2 -0
- package/dist/errors/options.js +13 -0
- package/dist/handlers/cancelHandler.d.ts +29 -0
- package/dist/handlers/cancelHandler.js +67 -0
- package/dist/handlers/completion-orchestrate.d.ts +41 -0
- package/dist/handlers/completion-orchestrate.js +190 -0
- package/dist/handlers/delete-cache.d.ts +11 -0
- package/dist/handlers/delete-cache.js +37 -0
- package/dist/handlers/download-asset.d.ts +2 -0
- package/dist/handlers/download-asset.js +146 -0
- package/dist/handlers/finetune.d.ts +2 -0
- package/dist/handlers/finetune.js +18 -0
- package/dist/handlers/get-loaded-model-info.d.ts +2 -0
- package/dist/handlers/get-loaded-model-info.js +45 -0
- package/dist/handlers/get-model-info.d.ts +2 -0
- package/dist/handlers/get-model-info.js +173 -0
- package/dist/handlers/get-system-resources.d.ts +2 -0
- package/dist/handlers/get-system-resources.js +53 -0
- package/dist/handlers/heartbeat.d.ts +2 -0
- package/dist/handlers/heartbeat.js +3 -0
- package/dist/handlers/load-model/download-manager.d.ts +95 -0
- package/dist/handlers/load-model/download-manager.js +306 -0
- package/dist/handlers/load-model/download-stats.d.ts +7 -0
- package/dist/handlers/load-model/download-stats.js +126 -0
- package/dist/handlers/load-model/handler.d.ts +2 -0
- package/dist/handlers/load-model/handler.js +287 -0
- package/dist/handlers/load-model/http.d.ts +3 -0
- package/dist/handlers/load-model/http.js +857 -0
- package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
- package/dist/handlers/load-model/hyperdrive.js +560 -0
- package/dist/handlers/load-model/index.d.ts +2 -0
- package/dist/handlers/load-model/index.js +2 -0
- package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/handlers/load-model/registry-client-options.js +18 -0
- package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/handlers/load-model/registry-companion-set.js +206 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
- package/dist/handlers/load-model/registry-download-utils.js +112 -0
- package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
- package/dist/handlers/load-model/registry-metadata.js +23 -0
- package/dist/handlers/load-model/registry.d.ts +4 -0
- package/dist/handlers/load-model/registry.js +298 -0
- package/dist/handlers/load-model/resolve-session.d.ts +33 -0
- package/dist/handlers/load-model/resolve-session.js +62 -0
- package/dist/handlers/load-model/resolve.d.ts +5 -0
- package/dist/handlers/load-model/resolve.js +158 -0
- package/dist/handlers/load-model/types.d.ts +59 -0
- package/dist/handlers/load-model/types.js +28 -0
- package/dist/handlers/logging-stream.d.ts +2 -0
- package/dist/handlers/logging-stream.js +39 -0
- package/dist/handlers/plugin-dispatch.d.ts +3 -0
- package/dist/handlers/plugin-dispatch.js +63 -0
- package/dist/handlers/plugin-invoke.d.ts +3 -0
- package/dist/handlers/plugin-invoke.js +81 -0
- package/dist/handlers/registry.d.ts +4 -0
- package/dist/handlers/registry.js +156 -0
- package/dist/handlers/resume.d.ts +2 -0
- package/dist/handlers/resume.js +11 -0
- package/dist/handlers/state.d.ts +2 -0
- package/dist/handlers/state.js +7 -0
- package/dist/handlers/suspend.d.ts +2 -0
- package/dist/handlers/suspend.js +11 -0
- package/dist/handlers/types.d.ts +13 -0
- package/dist/handlers/types.js +1 -0
- package/dist/handlers/unload-model.d.ts +2 -0
- package/dist/handlers/unload-model.js +29 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +12 -0
- package/dist/logging/addon.d.ts +5 -0
- package/dist/logging/addon.js +75 -0
- package/dist/logging/index.d.ts +5 -0
- package/dist/logging/index.js +4 -0
- package/dist/logging/logger.d.ts +41 -0
- package/dist/logging/logger.js +209 -0
- package/dist/logging/namespaces.d.ts +6 -0
- package/dist/logging/namespaces.js +7 -0
- package/dist/logging/types.d.ts +37 -0
- package/dist/logging/types.js +1 -0
- package/dist/logging/utils.d.ts +5 -0
- package/dist/logging/utils.js +113 -0
- package/dist/models/registry/index.d.ts +18 -0
- package/dist/models/registry/index.js +21 -0
- package/dist/models/registry/models.d.ts +22680 -0
- package/dist/models/registry/models.js +23908 -0
- package/dist/p2p/cancel-delegated.d.ts +3 -0
- package/dist/p2p/cancel-delegated.js +64 -0
- package/dist/p2p/completion-stream-delegated.d.ts +4 -0
- package/dist/p2p/completion-stream-delegated.js +51 -0
- package/dist/p2p/delegate-client.d.ts +9 -0
- package/dist/p2p/delegate-client.js +219 -0
- package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
- package/dist/p2p/delegate-connect-diagnostics.js +38 -0
- package/dist/p2p/delegate-transport.d.ts +18 -0
- package/dist/p2p/delegate-transport.js +237 -0
- package/dist/p2p/error-handlers.d.ts +4 -0
- package/dist/p2p/error-handlers.js +31 -0
- package/dist/p2p/handle-request.d.ts +2 -0
- package/dist/p2p/handle-request.js +171 -0
- package/dist/p2p/handler-utils.d.ts +19 -0
- package/dist/p2p/handler-utils.js +180 -0
- package/dist/p2p/heartbeat-delegated.d.ts +3 -0
- package/dist/p2p/heartbeat-delegated.js +32 -0
- package/dist/p2p/load-model-delegated.d.ts +6 -0
- package/dist/p2p/load-model-delegated.js +132 -0
- package/dist/p2p/progress-throttle.d.ts +7 -0
- package/dist/p2p/progress-throttle.js +47 -0
- package/dist/p2p/provideHandler/connection.d.ts +2 -0
- package/dist/p2p/provideHandler/connection.js +38 -0
- package/dist/p2p/provideHandler/handler.d.ts +2 -0
- package/dist/p2p/provideHandler/handler.js +66 -0
- package/dist/p2p/provideHandler/proxy.d.ts +2 -0
- package/dist/p2p/provideHandler/proxy.js +24 -0
- package/dist/p2p/provideHandler.d.ts +1 -0
- package/dist/p2p/provideHandler.js +2 -0
- package/dist/p2p/rpc-utils.d.ts +4 -0
- package/dist/p2p/rpc-utils.js +8 -0
- package/dist/p2p/stopProvideHandler.d.ts +2 -0
- package/dist/p2p/stopProvideHandler.js +29 -0
- package/dist/p2p/swarm.d.ts +9 -0
- package/dist/p2p/swarm.js +103 -0
- package/dist/p2p/transport-selector.d.ts +2 -0
- package/dist/p2p/transport-selector.js +18 -0
- package/dist/p2p/unload-model-delegated.d.ts +2 -0
- package/dist/p2p/unload-model-delegated.js +31 -0
- package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
- package/dist/plugins/builtin/asr-ggml/config.js +46 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
- package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
- package/dist/plugins/builtin/index.d.ts +11 -0
- package/dist/plugins/builtin/index.js +11 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
- package/dist/plugins/index.d.ts +9 -0
- package/dist/plugins/index.js +12 -0
- package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
- package/dist/plugins/ops/bci-transcribe.js +248 -0
- package/dist/plugins/ops/cancel.d.ts +25 -0
- package/dist/plugins/ops/cancel.js +43 -0
- package/dist/plugins/ops/embed.d.ts +15 -0
- package/dist/plugins/ops/embed.js +193 -0
- package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
- package/dist/plugins/ops/kv-cache-retention.js +121 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
- package/dist/plugins/ops/kv-cache-utils.js +120 -0
- package/dist/plugins/ops/load-model.d.ts +8 -0
- package/dist/plugins/ops/load-model.js +82 -0
- package/dist/plugins/ops/transcribe.d.ts +20 -0
- package/dist/plugins/ops/transcribe.js +343 -0
- package/dist/plugins/ops/translate.d.ts +7 -0
- package/dist/plugins/ops/translate.js +256 -0
- package/dist/plugins/ops/unload-model.d.ts +2 -0
- package/dist/plugins/ops/unload-model.js +45 -0
- package/dist/plugins/registry.d.ts +9 -0
- package/dist/plugins/registry.js +107 -0
- package/dist/profiling/aggregator.d.ts +14 -0
- package/dist/profiling/aggregator.js +100 -0
- package/dist/profiling/backend-diagnostics.d.ts +8 -0
- package/dist/profiling/backend-diagnostics.js +36 -0
- package/dist/profiling/clock.d.ts +8 -0
- package/dist/profiling/clock.js +23 -0
- package/dist/profiling/controller.d.ts +34 -0
- package/dist/profiling/controller.js +130 -0
- package/dist/profiling/delegation-profiler.d.ts +31 -0
- package/dist/profiling/delegation-profiler.js +155 -0
- package/dist/profiling/envelope.d.ts +10 -0
- package/dist/profiling/envelope.js +45 -0
- package/dist/profiling/events.d.ts +19 -0
- package/dist/profiling/events.js +35 -0
- package/dist/profiling/exporters.d.ts +9 -0
- package/dist/profiling/exporters.js +242 -0
- package/dist/profiling/index.d.ts +91 -0
- package/dist/profiling/index.js +87 -0
- package/dist/profiling/model-execution.d.ts +15 -0
- package/dist/profiling/model-execution.js +34 -0
- package/dist/profiling/operation-metrics.d.ts +24 -0
- package/dist/profiling/operation-metrics.js +370 -0
- package/dist/profiling/operation-wrappers.d.ts +17 -0
- package/dist/profiling/operation-wrappers.js +196 -0
- package/dist/profiling/profiler.d.ts +27 -0
- package/dist/profiling/profiler.js +127 -0
- package/dist/profiling/ring-buffer.d.ts +18 -0
- package/dist/profiling/ring-buffer.js +83 -0
- package/dist/profiling/types.d.ts +52 -0
- package/dist/profiling/types.js +1 -0
- package/dist/rag/handler.d.ts +2 -0
- package/dist/rag/handler.js +272 -0
- package/dist/rag/handlers/chunk.d.ts +2 -0
- package/dist/rag/handlers/chunk.js +7 -0
- package/dist/rag/handlers/close-workspace.d.ts +2 -0
- package/dist/rag/handlers/close-workspace.js +9 -0
- package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
- package/dist/rag/handlers/delete-embeddings.js +11 -0
- package/dist/rag/handlers/delete-workspace.d.ts +2 -0
- package/dist/rag/handlers/delete-workspace.js +14 -0
- package/dist/rag/handlers/ingest.d.ts +9 -0
- package/dist/rag/handlers/ingest.js +19 -0
- package/dist/rag/handlers/list-workspaces.d.ts +2 -0
- package/dist/rag/handlers/list-workspaces.js +4 -0
- package/dist/rag/handlers/reindex.d.ts +9 -0
- package/dist/rag/handlers/reindex.js +16 -0
- package/dist/rag/handlers/save-embeddings.d.ts +9 -0
- package/dist/rag/handlers/save-embeddings.js +15 -0
- package/dist/rag/handlers/search.d.ts +2 -0
- package/dist/rag/handlers/search.js +16 -0
- package/dist/rag/index.d.ts +11 -0
- package/dist/rag/index.js +11 -0
- package/dist/rag/rag-operation-manager.d.ts +28 -0
- package/dist/rag/rag-operation-manager.js +76 -0
- package/dist/rag/rag-workspace-manager.d.ts +14 -0
- package/dist/rag/rag-workspace-manager.js +143 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +166 -0
- package/dist/resources/collector.d.ts +643 -0
- package/dist/resources/collector.js +226 -0
- package/dist/resources/instance.d.ts +4 -0
- package/dist/resources/instance.js +15 -0
- package/dist/resources/native.d.ts +2 -0
- package/dist/resources/native.js +35 -0
- package/dist/resources/normalize.d.ts +12 -0
- package/dist/resources/normalize.js +73 -0
- package/dist/resources/types.d.ts +65 -0
- package/dist/resources/types.js +1 -0
- package/dist/runtime/cache-lock.d.ts +2 -0
- package/dist/runtime/cache-lock.js +67 -0
- package/dist/runtime/disposable-scope.d.ts +35 -0
- package/dist/runtime/disposable-scope.js +69 -0
- package/dist/runtime/env.d.ts +16 -0
- package/dist/runtime/env.js +53 -0
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +20 -0
- package/dist/runtime/lifecycle.js +102 -0
- package/dist/runtime/logging-stream-registry.d.ts +27 -0
- package/dist/runtime/logging-stream-registry.js +133 -0
- package/dist/runtime/model-config-utils.d.ts +14 -0
- package/dist/runtime/model-config-utils.js +128 -0
- package/dist/runtime/model-registry.d.ts +74 -0
- package/dist/runtime/model-registry.js +130 -0
- package/dist/runtime/registry-client.d.ts +4 -0
- package/dist/runtime/registry-client.js +118 -0
- package/dist/runtime/request-context.d.ts +72 -0
- package/dist/runtime/request-context.js +114 -0
- package/dist/runtime/request-id.d.ts +17 -0
- package/dist/runtime/request-id.js +29 -0
- package/dist/runtime/request-registry.d.ts +194 -0
- package/dist/runtime/request-registry.js +546 -0
- package/dist/runtime/runtime-lifecycle.d.ts +37 -0
- package/dist/runtime/runtime-lifecycle.js +163 -0
- package/dist/runtime/state.d.ts +26 -0
- package/dist/runtime/state.js +138 -0
- package/dist/schemas/audio-gen.d.ts +252 -0
- package/dist/schemas/audio-gen.js +69 -0
- package/dist/schemas/batch-completion-stream.d.ts +467 -0
- package/dist/schemas/batch-completion-stream.js +123 -0
- package/dist/schemas/bci-config.d.ts +74 -0
- package/dist/schemas/bci-config.js +58 -0
- package/dist/schemas/bci.d.ts +178 -0
- package/dist/schemas/bci.js +89 -0
- package/dist/schemas/cancel.d.ts +203 -0
- package/dist/schemas/cancel.js +135 -0
- package/dist/schemas/classification.d.ts +52 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +6420 -0
- package/dist/schemas/common.js +133 -0
- package/dist/schemas/completion-event.d.ts +248 -0
- package/dist/schemas/completion-event.js +94 -0
- package/dist/schemas/completion-stream.d.ts +475 -0
- package/dist/schemas/completion-stream.js +236 -0
- package/dist/schemas/config.d.ts +5022 -0
- package/dist/schemas/config.js +169 -0
- package/dist/schemas/delegate.d.ts +29 -0
- package/dist/schemas/delegate.js +40 -0
- package/dist/schemas/download-asset.d.ts +225 -0
- package/dist/schemas/download-asset.js +71 -0
- package/dist/schemas/embed.d.ts +41 -0
- package/dist/schemas/embed.js +35 -0
- package/dist/schemas/engine-addon-map.d.ts +20 -0
- package/dist/schemas/engine-addon-map.js +78 -0
- package/dist/schemas/error.d.ts +40 -0
- package/dist/schemas/error.js +65 -0
- package/dist/schemas/errors.d.ts +119 -0
- package/dist/schemas/errors.js +615 -0
- package/dist/schemas/finetune.d.ts +621 -0
- package/dist/schemas/finetune.js +163 -0
- package/dist/schemas/index.d.ts +45 -0
- package/dist/schemas/index.js +45 -0
- package/dist/schemas/json-schema.d.ts +15 -0
- package/dist/schemas/json-schema.js +2 -0
- package/dist/schemas/lifecycle.d.ts +38 -0
- package/dist/schemas/lifecycle.js +21 -0
- package/dist/schemas/llamacpp-config.d.ts +403 -0
- package/dist/schemas/llamacpp-config.js +133 -0
- package/dist/schemas/load-model.d.ts +25635 -0
- package/dist/schemas/load-model.js +629 -0
- package/dist/schemas/logging-stream.d.ts +33 -0
- package/dist/schemas/logging-stream.js +17 -0
- package/dist/schemas/mcp-adapter.d.ts +46 -0
- package/dist/schemas/mcp-adapter.js +1 -0
- package/dist/schemas/model-file.d.ts +27 -0
- package/dist/schemas/model-file.js +29 -0
- package/dist/schemas/model-info.d.ts +197 -0
- package/dist/schemas/model-info.js +144 -0
- package/dist/schemas/model-ops.d.ts +36 -0
- package/dist/schemas/model-ops.js +40 -0
- package/dist/schemas/model-src-utils.d.ts +192 -0
- package/dist/schemas/model-src-utils.js +61 -0
- package/dist/schemas/model-types.d.ts +275 -0
- package/dist/schemas/model-types.js +223 -0
- package/dist/schemas/ocr.d.ts +138 -0
- package/dist/schemas/ocr.js +59 -0
- package/dist/schemas/path-validation.d.ts +9 -0
- package/dist/schemas/path-validation.js +22 -0
- package/dist/schemas/plugin.d.ts +341 -0
- package/dist/schemas/plugin.js +241 -0
- package/dist/schemas/profiling.d.ts +695 -0
- package/dist/schemas/profiling.js +100 -0
- package/dist/schemas/provide.d.ts +50 -0
- package/dist/schemas/provide.js +32 -0
- package/dist/schemas/rag.d.ts +301 -0
- package/dist/schemas/rag.js +211 -0
- package/dist/schemas/registry.d.ts +453 -0
- package/dist/schemas/registry.js +104 -0
- package/dist/schemas/reload-config.d.ts +585 -0
- package/dist/schemas/reload-config.js +42 -0
- package/dist/schemas/runtime-context.d.ts +18 -0
- package/dist/schemas/runtime-context.js +7 -0
- package/dist/schemas/sdcpp-config.d.ts +1126 -0
- package/dist/schemas/sdcpp-config.js +748 -0
- package/dist/schemas/system-resources.d.ts +3966 -0
- package/dist/schemas/system-resources.js +149 -0
- package/dist/schemas/text-to-speech.d.ts +1352 -0
- package/dist/schemas/text-to-speech.js +364 -0
- package/dist/schemas/tools.d.ts +80 -0
- package/dist/schemas/tools.js +52 -0
- package/dist/schemas/transcription-config.d.ts +161 -0
- package/dist/schemas/transcription-config.js +156 -0
- package/dist/schemas/transcription.d.ts +261 -0
- package/dist/schemas/transcription.js +145 -0
- package/dist/schemas/translate.d.ts +138 -0
- package/dist/schemas/translate.js +103 -0
- package/dist/schemas/translation-config.d.ts +1031 -0
- package/dist/schemas/translation-config.js +170 -0
- package/dist/schemas/vla.d.ts +110 -0
- package/dist/schemas/vla.js +151 -0
- package/dist/selection.d.ts +8 -0
- package/dist/selection.js +14 -0
- package/dist/surface.d.ts +18 -0
- package/dist/surface.js +30 -0
- package/dist/utils/addon-responses.d.ts +58 -0
- package/dist/utils/addon-responses.js +1 -0
- package/dist/utils/aggregate-events.d.ts +25 -0
- package/dist/utils/aggregate-events.js +73 -0
- package/dist/utils/archive.d.ts +4 -0
- package/dist/utils/archive.js +180 -0
- package/dist/utils/asr-events.d.ts +14 -0
- package/dist/utils/asr-events.js +30 -0
- package/dist/utils/audio/decoder.d.ts +4 -0
- package/dist/utils/audio/decoder.js +80 -0
- package/dist/utils/audio-input.d.ts +9 -0
- package/dist/utils/audio-input.js +34 -0
- package/dist/utils/cache/auto-history.d.ts +3 -0
- package/dist/utils/cache/auto-history.js +26 -0
- package/dist/utils/cache/paths.d.ts +42 -0
- package/dist/utils/cache/paths.js +133 -0
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/cache/types.js +1 -0
- package/dist/utils/cache-normalize.d.ts +9 -0
- package/dist/utils/cache-normalize.js +20 -0
- package/dist/utils/checksum.d.ts +3 -0
- package/dist/utils/checksum.js +29 -0
- package/dist/utils/clear-storage.d.ts +4 -0
- package/dist/utils/clear-storage.js +14 -0
- package/dist/utils/completion-normalizer.d.ts +17 -0
- package/dist/utils/completion-normalizer.js +453 -0
- package/dist/utils/config-transform.d.ts +2 -0
- package/dist/utils/config-transform.js +21 -0
- package/dist/utils/decorate-promise.d.ts +33 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/encoding.d.ts +16 -0
- package/dist/utils/encoding.js +27 -0
- package/dist/utils/expand-gguf-shards.d.ts +2 -0
- package/dist/utils/expand-gguf-shards.js +22 -0
- package/dist/utils/formatting.d.ts +18 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
- package/dist/utils/gguf-tensor-extractor.js +311 -0
- package/dist/utils/global-singleton.d.ts +1 -0
- package/dist/utils/global-singleton.js +10 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/load-model-validation.d.ts +6 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/dist/utils/mcp-adapter.d.ts +11 -0
- package/dist/utils/mcp-adapter.js +96 -0
- package/dist/utils/multi-gpu-mobile.d.ts +4 -0
- package/dist/utils/multi-gpu-mobile.js +10 -0
- package/dist/utils/object.d.ts +1 -0
- package/dist/utils/object.js +3 -0
- package/dist/utils/path-security.d.ts +22 -0
- package/dist/utils/path-security.js +69 -0
- package/dist/utils/qvac-paths.d.ts +1 -0
- package/dist/utils/qvac-paths.js +5 -0
- package/dist/utils/response-format.d.ts +2 -0
- package/dist/utils/response-format.js +20 -0
- package/dist/utils/shard-utils.d.ts +47 -0
- package/dist/utils/shard-utils.js +123 -0
- package/dist/utils/tool-helpers.d.ts +24 -0
- package/dist/utils/tool-helpers.js +111 -0
- package/dist/utils/tool-integration.d.ts +23 -0
- package/dist/utils/tool-integration.js +29 -0
- package/dist/utils/tools/dialect.d.ts +7 -0
- package/dist/utils/tools/dialect.js +20 -0
- package/dist/utils/tools/index.d.ts +2 -0
- package/dist/utils/tools/index.js +2 -0
- package/dist/utils/tools/parser.d.ts +5 -0
- package/dist/utils/tools/parser.js +69 -0
- package/dist/utils/tools/parsers/dsml.d.ts +3 -0
- package/dist/utils/tools/parsers/dsml.js +177 -0
- package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
- package/dist/utils/tools/parsers/gemma4native.js +65 -0
- package/dist/utils/tools/parsers/harmony.d.ts +3 -0
- package/dist/utils/tools/parsers/harmony.js +57 -0
- package/dist/utils/tools/parsers/hermes.d.ts +3 -0
- package/dist/utils/tools/parsers/hermes.js +122 -0
- package/dist/utils/tools/parsers/json.d.ts +5 -0
- package/dist/utils/tools/parsers/json.js +115 -0
- package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
- package/dist/utils/tools/parsers/pythonic.js +267 -0
- package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
- package/dist/utils/tools/parsers/qwen35.js +125 -0
- package/dist/utils/tools/shared.d.ts +17 -0
- package/dist/utils/tools/shared.js +52 -0
- package/dist/utils/transcribe-metadata.d.ts +20 -0
- package/dist/utils/transcribe-metadata.js +28 -0
- package/dist/utils/tts-stats.d.ts +25 -0
- package/dist/utils/tts-stats.js +16 -0
- package/dist/utils/withTimeout.d.ts +17 -0
- package/dist/utils/withTimeout.js +47 -0
- package/dist/utils/zod-error.d.ts +2 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +266 -1
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { CancelRequest, CancelResponse } from '../schemas/cancel.js';
|
|
2
|
+
/**
|
|
3
|
+
* Cancel handler entry point. Every long-running handler registers
|
|
4
|
+
* itself on the process-singleton `RequestRegistry`, so the cancel
|
|
5
|
+
* surface is two paths that route through the same registry primitive:
|
|
6
|
+
*
|
|
7
|
+
* - `{ operation: "request", requestId, clearCache? }` — targeted
|
|
8
|
+
* cancel by caller-generated id. Looks up the registry entry,
|
|
9
|
+
* fires its abort signal, optionally marks the underlying download
|
|
10
|
+
* transfer for cache clear. The "stop-button race" case (a cancel
|
|
11
|
+
* beats its own begin) is handled inside the
|
|
12
|
+
* registry via the cancel-before-begin tripwire.
|
|
13
|
+
*
|
|
14
|
+
* - `{ operation: "broad", modelId, kind? }` — abort every in-flight
|
|
15
|
+
* request on a model (optionally narrowed by `kind`). Used for
|
|
16
|
+
* model unload, app shutdown, and admin sweeps where the caller
|
|
17
|
+
* has no `requestId`. Delegates to the `cancel` bare op so the
|
|
18
|
+
* `ModelNotLoadedError` validation is shared with internal
|
|
19
|
+
* engine-side broad cancels.
|
|
20
|
+
*
|
|
21
|
+
* Always returns `success: true` plus a `cancelled` count (the number
|
|
22
|
+
* of contexts this call flipped to `cancelling` — already-cancelled
|
|
23
|
+
* contexts are not counted). A targeted cancel with no in-flight
|
|
24
|
+
* match still returns `success: true, cancelled: 0`; the
|
|
25
|
+
* cancel-before-begin tripwire ensures the cancel is applied
|
|
26
|
+
* retroactively if a matching begin arrives within the registry's
|
|
27
|
+
* race window.
|
|
28
|
+
*/
|
|
29
|
+
export declare function cancelHandler(request: CancelRequest): CancelResponse;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { cancel as cancelByModelId } from '../plugins/ops/cancel.js';
|
|
2
|
+
import { getRequestRegistry } from '../runtime/index.js';
|
|
3
|
+
import { markClearCacheForRequest } from '../handlers/load-model/download-manager.js';
|
|
4
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
5
|
+
const logger = getEngineLogger();
|
|
6
|
+
/**
|
|
7
|
+
* Cancel handler entry point. Every long-running handler registers
|
|
8
|
+
* itself on the process-singleton `RequestRegistry`, so the cancel
|
|
9
|
+
* surface is two paths that route through the same registry primitive:
|
|
10
|
+
*
|
|
11
|
+
* - `{ operation: "request", requestId, clearCache? }` — targeted
|
|
12
|
+
* cancel by caller-generated id. Looks up the registry entry,
|
|
13
|
+
* fires its abort signal, optionally marks the underlying download
|
|
14
|
+
* transfer for cache clear. The "stop-button race" case (a cancel
|
|
15
|
+
* beats its own begin) is handled inside the
|
|
16
|
+
* registry via the cancel-before-begin tripwire.
|
|
17
|
+
*
|
|
18
|
+
* - `{ operation: "broad", modelId, kind? }` — abort every in-flight
|
|
19
|
+
* request on a model (optionally narrowed by `kind`). Used for
|
|
20
|
+
* model unload, app shutdown, and admin sweeps where the caller
|
|
21
|
+
* has no `requestId`. Delegates to the `cancel` bare op so the
|
|
22
|
+
* `ModelNotLoadedError` validation is shared with internal
|
|
23
|
+
* engine-side broad cancels.
|
|
24
|
+
*
|
|
25
|
+
* Always returns `success: true` plus a `cancelled` count (the number
|
|
26
|
+
* of contexts this call flipped to `cancelling` — already-cancelled
|
|
27
|
+
* contexts are not counted). A targeted cancel with no in-flight
|
|
28
|
+
* match still returns `success: true, cancelled: 0`; the
|
|
29
|
+
* cancel-before-begin tripwire ensures the cancel is applied
|
|
30
|
+
* retroactively if a matching begin arrives within the registry's
|
|
31
|
+
* race window.
|
|
32
|
+
*/
|
|
33
|
+
export function cancelHandler(request) {
|
|
34
|
+
try {
|
|
35
|
+
if (request.operation === 'request') {
|
|
36
|
+
if (request.clearCache) {
|
|
37
|
+
markClearCacheForRequest(request.requestId);
|
|
38
|
+
}
|
|
39
|
+
const cancelled = getRequestRegistry().cancel({
|
|
40
|
+
requestId: request.requestId
|
|
41
|
+
});
|
|
42
|
+
if (cancelled === 0) {
|
|
43
|
+
// info-level (not debug) because the decorated-promise pattern
|
|
44
|
+
// makes "no in-flight match" a common and user-visible case:
|
|
45
|
+
// a Stop button fired after the request settled but before
|
|
46
|
+
// the UI cleared lands here. The cancel-before-begin tripwire
|
|
47
|
+
// inside the registry already captured the cancel for any
|
|
48
|
+
// matching begin in flight; this log just helps operators
|
|
49
|
+
// debugging "my Stop button isn't working" without lowering
|
|
50
|
+
// the log level.
|
|
51
|
+
logger.info(`[cancel] no in-flight request matched requestId=${request.requestId}`);
|
|
52
|
+
}
|
|
53
|
+
return { type: 'cancel', success: true, cancelled };
|
|
54
|
+
}
|
|
55
|
+
// operation === "broad"
|
|
56
|
+
const cancelled = cancelByModelId({ modelId: request.modelId }, request.kind ? { kind: request.kind } : undefined);
|
|
57
|
+
return { type: 'cancel', success: true, cancelled };
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
logger.error('Error during cancellation:', error);
|
|
61
|
+
return {
|
|
62
|
+
type: 'cancel',
|
|
63
|
+
success: false,
|
|
64
|
+
error: error instanceof Error ? error.message : 'Unknown error'
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { CompletionOrchestrateRequest, CompletionOrchestrateResponse, CompletionStreamRequest, CompletionStreamResponse } from '../schemas/index.js';
|
|
2
|
+
import { type ToolCallbackResult } from '../schemas/completion-stream.js';
|
|
3
|
+
/**
|
|
4
|
+
* Reads newline-delimited JSON tool results off the duplex request stream
|
|
5
|
+
* and resolves per-callId waiters. Chunk boundaries don't align with lines,
|
|
6
|
+
* so a partial tail is buffered until its newline arrives.
|
|
7
|
+
*
|
|
8
|
+
* The pump is torn down by `close()` (called from a `finally` when
|
|
9
|
+
* orchestration ends, however it ends) so the upstream iterator is returned
|
|
10
|
+
* and the reader stops holding the request stream open.
|
|
11
|
+
*/
|
|
12
|
+
export declare class ToolResultReader {
|
|
13
|
+
private buffer;
|
|
14
|
+
private readonly results;
|
|
15
|
+
private readonly waiters;
|
|
16
|
+
private failed;
|
|
17
|
+
private closed;
|
|
18
|
+
private readonly iterator;
|
|
19
|
+
constructor(inputStream: AsyncIterable<Buffer>);
|
|
20
|
+
private pump;
|
|
21
|
+
private failWaiters;
|
|
22
|
+
private accept;
|
|
23
|
+
waitFor(callId: string, timeoutMs?: number): Promise<ToolCallbackResult>;
|
|
24
|
+
/**
|
|
25
|
+
* Stop the pump and release the upstream iterator. Idempotent. Pending
|
|
26
|
+
* waiters are failed so a caller blocked in `waitFor` can't hang after the
|
|
27
|
+
* stream is torn down.
|
|
28
|
+
*/
|
|
29
|
+
close(): void;
|
|
30
|
+
}
|
|
31
|
+
type RunTurn = (request: CompletionStreamRequest) => AsyncGenerator<CompletionStreamResponse, void, unknown>;
|
|
32
|
+
/**
|
|
33
|
+
* The orchestration core, pure over its collaborators so it unit-tests
|
|
34
|
+
* without the plugin registry: runs completion turns, forwards their events
|
|
35
|
+
* downstream, and when a turn requests tool calls, emits `toolCallback`
|
|
36
|
+
* frames and blocks on the client's results before starting the next turn
|
|
37
|
+
* with the extended history.
|
|
38
|
+
*/
|
|
39
|
+
export declare function orchestrateCompletion(request: CompletionOrchestrateRequest, runTurn: RunTurn, reader: Pick<ToolResultReader, 'waitFor'>): AsyncGenerator<CompletionOrchestrateResponse>;
|
|
40
|
+
export declare function handleCompletionOrchestrate(request: CompletionOrchestrateRequest, inputStream: AsyncIterable<Buffer>): AsyncGenerator<CompletionOrchestrateResponse>;
|
|
41
|
+
export {};
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { toolCallbackResultSchema } from '../schemas/completion-stream.js';
|
|
2
|
+
import { aggregateEvents } from '../utils/aggregate-events.js';
|
|
3
|
+
import { CompletionFailedError } from '../errors/index.js';
|
|
4
|
+
import { dispatchPluginStream } from '../handlers/plugin-dispatch.js';
|
|
5
|
+
const DEFAULT_MAX_TOOL_TURNS = 8;
|
|
6
|
+
// Generous ceiling on one client-side tool execution: long enough for a slow
|
|
7
|
+
// MCP round trip, short enough that a client that stopped reading can't pin a
|
|
8
|
+
// model slot forever.
|
|
9
|
+
const TOOL_RESULT_TIMEOUT_MS = 5 * 60 * 1000;
|
|
10
|
+
// A single tool result is small JSON; anything past this without a newline is a
|
|
11
|
+
// malformed/hostile client trying to grow the buffer without bound. Fail loudly
|
|
12
|
+
// instead of accumulating forever.
|
|
13
|
+
const MAX_TOOL_RESULT_LINE_BYTES = 16 * 1024 * 1024;
|
|
14
|
+
/**
|
|
15
|
+
* Reads newline-delimited JSON tool results off the duplex request stream
|
|
16
|
+
* and resolves per-callId waiters. Chunk boundaries don't align with lines,
|
|
17
|
+
* so a partial tail is buffered until its newline arrives.
|
|
18
|
+
*
|
|
19
|
+
* The pump is torn down by `close()` (called from a `finally` when
|
|
20
|
+
* orchestration ends, however it ends) so the upstream iterator is returned
|
|
21
|
+
* and the reader stops holding the request stream open.
|
|
22
|
+
*/
|
|
23
|
+
export class ToolResultReader {
|
|
24
|
+
buffer = '';
|
|
25
|
+
results = new Map();
|
|
26
|
+
waiters = new Map();
|
|
27
|
+
failed;
|
|
28
|
+
closed = false;
|
|
29
|
+
iterator;
|
|
30
|
+
constructor(inputStream) {
|
|
31
|
+
this.iterator = inputStream[Symbol.asyncIterator]();
|
|
32
|
+
void this.pump();
|
|
33
|
+
}
|
|
34
|
+
async pump() {
|
|
35
|
+
try {
|
|
36
|
+
while (!this.closed) {
|
|
37
|
+
const next = await this.iterator.next();
|
|
38
|
+
if (next.done)
|
|
39
|
+
break;
|
|
40
|
+
this.buffer += next.value.toString();
|
|
41
|
+
let nl;
|
|
42
|
+
while ((nl = this.buffer.indexOf('\n')) !== -1) {
|
|
43
|
+
const line = this.buffer.slice(0, nl);
|
|
44
|
+
this.buffer = this.buffer.slice(nl + 1);
|
|
45
|
+
if (line.trim())
|
|
46
|
+
this.accept(line);
|
|
47
|
+
}
|
|
48
|
+
if (this.buffer.length > MAX_TOOL_RESULT_LINE_BYTES) {
|
|
49
|
+
throw new CompletionFailedError('tool result frame exceeds the maximum size');
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (!this.closed && this.buffer.trim())
|
|
53
|
+
this.accept(this.buffer);
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
this.failWaiters(error instanceof Error ? error : new Error(String(error)));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
failWaiters(error) {
|
|
60
|
+
this.failed = error;
|
|
61
|
+
// Wake every waiter; each rethrows via waitFor's failure check.
|
|
62
|
+
for (const [callId, resolve] of this.waiters) {
|
|
63
|
+
this.waiters.delete(callId);
|
|
64
|
+
resolve({ callId, error: error.message });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
accept(line) {
|
|
68
|
+
const parsed = toolCallbackResultSchema.parse(JSON.parse(line));
|
|
69
|
+
const waiter = this.waiters.get(parsed.callId);
|
|
70
|
+
if (waiter) {
|
|
71
|
+
this.waiters.delete(parsed.callId);
|
|
72
|
+
waiter(parsed);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
this.results.set(parsed.callId, parsed);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async waitFor(callId, timeoutMs = TOOL_RESULT_TIMEOUT_MS) {
|
|
79
|
+
const buffered = this.results.get(callId);
|
|
80
|
+
if (buffered) {
|
|
81
|
+
this.results.delete(callId);
|
|
82
|
+
return buffered;
|
|
83
|
+
}
|
|
84
|
+
if (this.failed)
|
|
85
|
+
throw this.failed;
|
|
86
|
+
return await new Promise((resolve, reject) => {
|
|
87
|
+
const timer = setTimeout(() => {
|
|
88
|
+
this.waiters.delete(callId);
|
|
89
|
+
reject(new CompletionFailedError(`tool callback ${callId} timed out`));
|
|
90
|
+
}, timeoutMs);
|
|
91
|
+
this.waiters.set(callId, (result) => {
|
|
92
|
+
clearTimeout(timer);
|
|
93
|
+
resolve(result);
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Stop the pump and release the upstream iterator. Idempotent. Pending
|
|
99
|
+
* waiters are failed so a caller blocked in `waitFor` can't hang after the
|
|
100
|
+
* stream is torn down.
|
|
101
|
+
*/
|
|
102
|
+
close() {
|
|
103
|
+
if (this.closed)
|
|
104
|
+
return;
|
|
105
|
+
this.closed = true;
|
|
106
|
+
this.buffer = '';
|
|
107
|
+
this.results.clear();
|
|
108
|
+
if (!this.failed)
|
|
109
|
+
this.failWaiters(new CompletionFailedError('tool result stream closed'));
|
|
110
|
+
void this.iterator.return?.();
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* The orchestration core, pure over its collaborators so it unit-tests
|
|
115
|
+
* without the plugin registry: runs completion turns, forwards their events
|
|
116
|
+
* downstream, and when a turn requests tool calls, emits `toolCallback`
|
|
117
|
+
* frames and blocks on the client's results before starting the next turn
|
|
118
|
+
* with the extended history.
|
|
119
|
+
*/
|
|
120
|
+
export async function* orchestrateCompletion(request, runTurn, reader) {
|
|
121
|
+
// Pull the loop bound out of the per-turn base; everything else (including
|
|
122
|
+
// `requestId`) flows into each inner completionStream turn. Threading the
|
|
123
|
+
// orchestrate `requestId` down is what makes `cancel({ requestId })` reach
|
|
124
|
+
// the turn that's currently generating — the expensive part a Stop button
|
|
125
|
+
// targets. Turns run strictly sequentially: turn N's request-registry entry
|
|
126
|
+
// is disposed (on the `for await` break below) before turn N+1 calls
|
|
127
|
+
// `begin(...)`, so reusing the id can't collide in `entries`. A cancel that
|
|
128
|
+
// lands between turns (during a tool-result wait) is caught by the
|
|
129
|
+
// registry's cancel-before-begin tripwire and aborts the next turn's
|
|
130
|
+
// `begin(...)`. Each turn's `history` and `type` are set below.
|
|
131
|
+
const { maxToolTurns, ...turnBase } = request;
|
|
132
|
+
const maxTurns = maxToolTurns ?? DEFAULT_MAX_TOOL_TURNS;
|
|
133
|
+
let history = [...request.history];
|
|
134
|
+
for (let turn = 0; turn < maxTurns; turn++) {
|
|
135
|
+
const turnEvents = [];
|
|
136
|
+
const innerRequest = {
|
|
137
|
+
...turnBase,
|
|
138
|
+
history,
|
|
139
|
+
type: 'completionStream'
|
|
140
|
+
};
|
|
141
|
+
for await (const response of runTurn(innerRequest)) {
|
|
142
|
+
turnEvents.push(...response.events);
|
|
143
|
+
yield { type: 'completionOrchestrate', turn, events: response.events };
|
|
144
|
+
if (response.done)
|
|
145
|
+
break;
|
|
146
|
+
}
|
|
147
|
+
const { toolCalls, rawFullText, contentText, error, cancelled } = aggregateEvents(turnEvents);
|
|
148
|
+
// Terminal turn: the model answered (or failed/was cancelled) without
|
|
149
|
+
// requesting tools — the client folds the forwarded events itself.
|
|
150
|
+
if (error || cancelled || toolCalls.length === 0) {
|
|
151
|
+
yield { type: 'completionOrchestrate', done: true };
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
// The assistant's tool-call turn goes back verbatim so the model sees
|
|
155
|
+
// its own call syntax; each result follows as a `tool` message.
|
|
156
|
+
history = [...history, { role: 'assistant', content: rawFullText ?? contentText }];
|
|
157
|
+
for (const call of toolCalls) {
|
|
158
|
+
yield {
|
|
159
|
+
type: 'completionOrchestrate',
|
|
160
|
+
turn,
|
|
161
|
+
toolCallback: { callId: call.id, name: call.name, arguments: call.arguments }
|
|
162
|
+
};
|
|
163
|
+
const result = await reader.waitFor(call.id);
|
|
164
|
+
const content = result.error !== undefined
|
|
165
|
+
? JSON.stringify({ error: result.error })
|
|
166
|
+
: JSON.stringify(result.result ?? null);
|
|
167
|
+
history = [...history, { role: 'tool', content }];
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
// Turn cap reached without a natural end. Emit a terminal done frame with a
|
|
171
|
+
// stopReason so the client can tell this was truncated (the model still
|
|
172
|
+
// wanted a tool) rather than a natural finish; the last turn's events carry
|
|
173
|
+
// whatever the model produced.
|
|
174
|
+
yield { type: 'completionOrchestrate', done: true, stopReason: 'maxToolTurns' };
|
|
175
|
+
}
|
|
176
|
+
export async function* handleCompletionOrchestrate(request, inputStream) {
|
|
177
|
+
const reader = new ToolResultReader(inputStream);
|
|
178
|
+
const runTurn = async function* (innerRequest) {
|
|
179
|
+
yield* dispatchPluginStream(innerRequest.modelId, 'completionStream', innerRequest);
|
|
180
|
+
};
|
|
181
|
+
try {
|
|
182
|
+
yield* orchestrateCompletion(request, runTurn, reader);
|
|
183
|
+
}
|
|
184
|
+
finally {
|
|
185
|
+
// Tear the reader down however orchestration ends -- natural finish, turn
|
|
186
|
+
// cap, error, or the client abandoning the stream -- so the pump stops and
|
|
187
|
+
// the request stream isn't held open.
|
|
188
|
+
reader.close();
|
|
189
|
+
}
|
|
190
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { DeleteCacheRequest, DeleteCacheResponse } from '../schemas/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* RPC handler for `deleteCache(...)`. The three KV-cache bookkeeping
|
|
4
|
+
* layers (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`)
|
|
5
|
+
* are private to `kv-cache-session.ts`; this handler delegates to that
|
|
6
|
+
* module's single administrative entry point (`deleteKvCacheState`).
|
|
7
|
+
* The handler must have **zero** direct references to
|
|
8
|
+
* `fsPromises.unlink`, the `initializedCaches` set, or the
|
|
9
|
+
* `cachedMessageCounts` map.
|
|
10
|
+
*/
|
|
11
|
+
export declare function handleDeleteCache(request: DeleteCacheRequest): Promise<DeleteCacheResponse>;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { deleteKvCacheState } from '../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
|
|
2
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
3
|
+
const logger = getEngineLogger();
|
|
4
|
+
/**
|
|
5
|
+
* RPC handler for `deleteCache(...)`. The three KV-cache bookkeeping
|
|
6
|
+
* layers (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`)
|
|
7
|
+
* are private to `kv-cache-session.ts`; this handler delegates to that
|
|
8
|
+
* module's single administrative entry point (`deleteKvCacheState`).
|
|
9
|
+
* The handler must have **zero** direct references to
|
|
10
|
+
* `fsPromises.unlink`, the `initializedCaches` set, or the
|
|
11
|
+
* `cachedMessageCounts` map.
|
|
12
|
+
*/
|
|
13
|
+
export async function handleDeleteCache(request) {
|
|
14
|
+
try {
|
|
15
|
+
if ('all' in request && request.all) {
|
|
16
|
+
await deleteKvCacheState({ all: true });
|
|
17
|
+
}
|
|
18
|
+
else if ('kvCacheKey' in request) {
|
|
19
|
+
await deleteKvCacheState({
|
|
20
|
+
kvCacheKey: request.kvCacheKey,
|
|
21
|
+
...(request.modelId !== undefined && { modelId: request.modelId })
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
type: 'deleteCache',
|
|
26
|
+
success: true
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
logger.error('Error deleting cache:', error);
|
|
31
|
+
return {
|
|
32
|
+
type: 'deleteCache',
|
|
33
|
+
success: false,
|
|
34
|
+
error: error instanceof Error ? error.message : 'Unknown error'
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
53
|
+
import { PROFILING_KEY, OPERATION_EVENT_KEY } from '../schemas/index.js';
|
|
54
|
+
import { resolveModelPath, resolveModelPathWithStats } from '../handlers/load-model/resolve.js';
|
|
55
|
+
import { buildDownloadProfilingFields } from '../handlers/load-model/types.js';
|
|
56
|
+
import { nowMs, generateProfileId } from '../profiling/clock.js';
|
|
57
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
58
|
+
import { getRequestRegistry, withRequestContext } from '../runtime/index.js';
|
|
59
|
+
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
60
|
+
import { InferenceCancelledError } from '../errors/index.js';
|
|
61
|
+
const logger = getEngineLogger();
|
|
62
|
+
export async function handleDownloadAsset(request, progressCallback) {
|
|
63
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
64
|
+
try {
|
|
65
|
+
const { assetSrc, seed } = request;
|
|
66
|
+
const profilingMeta = request[PROFILING_KEY];
|
|
67
|
+
const profilingEnabled = profilingMeta?.enabled !== false && !!profilingMeta;
|
|
68
|
+
const requestId = request.requestId ?? generateRandomRequestId();
|
|
69
|
+
// `downloadAsset` is artifact-shaped, not model-shaped — there is no
|
|
70
|
+
// `modelId` to register on the registry entry. Cancel by `requestId`
|
|
71
|
+
// is the primary path; `cancel({ modelId })` is intentionally a
|
|
72
|
+
// non-match for this kind.
|
|
73
|
+
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
74
|
+
requestId,
|
|
75
|
+
kind: 'downloadAsset'
|
|
76
|
+
}), true);
|
|
77
|
+
const log = withRequestContext(getEngineLogger(), ctx);
|
|
78
|
+
log.debug(`downloadAsset start assetSrc=${assetSrc}`);
|
|
79
|
+
const hooks = {
|
|
80
|
+
requestBinding: {
|
|
81
|
+
signal: ctx.signal,
|
|
82
|
+
scope: ctx.scope,
|
|
83
|
+
requestId
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
try {
|
|
87
|
+
const totalDownloadStart = profilingEnabled ? nowMs() : 0;
|
|
88
|
+
let sourceType;
|
|
89
|
+
let downloadStats;
|
|
90
|
+
if (profilingEnabled) {
|
|
91
|
+
const result = await resolveModelPathWithStats(assetSrc, progressCallback, seed, ctx.signal, hooks);
|
|
92
|
+
sourceType = result.sourceType;
|
|
93
|
+
downloadStats = result.downloadStats;
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
await resolveModelPath(assetSrc, progressCallback, seed, ctx.signal, hooks);
|
|
97
|
+
}
|
|
98
|
+
const response = {
|
|
99
|
+
type: 'downloadAsset',
|
|
100
|
+
success: true,
|
|
101
|
+
assetId: assetSrc
|
|
102
|
+
};
|
|
103
|
+
if (profilingEnabled) {
|
|
104
|
+
const totalDownloadTimeMs = nowMs() - totalDownloadStart;
|
|
105
|
+
const profileId = profilingMeta?.id ?? generateProfileId();
|
|
106
|
+
const { gauges, tags } = buildDownloadProfilingFields(downloadStats, sourceType);
|
|
107
|
+
gauges['totalDownloadTime'] = totalDownloadTimeMs;
|
|
108
|
+
const operationEvent = {
|
|
109
|
+
op: 'downloadAsset',
|
|
110
|
+
kind: 'handler',
|
|
111
|
+
ms: totalDownloadTimeMs,
|
|
112
|
+
profileId,
|
|
113
|
+
gauges: Object.keys(gauges).length > 0 ? gauges : undefined,
|
|
114
|
+
tags: Object.keys(tags).length > 0 ? tags : undefined
|
|
115
|
+
};
|
|
116
|
+
response[OPERATION_EVENT_KEY] = operationEvent;
|
|
117
|
+
}
|
|
118
|
+
return response;
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
// Mirror the load-model handler's cancel contract: a typed cancel
|
|
122
|
+
// bubbles up as the rejection so client-side callers can branch on
|
|
123
|
+
// `instanceof InferenceCancelledError`. Every other error keeps
|
|
124
|
+
// the legacy `success: false` envelope.
|
|
125
|
+
if (error instanceof InferenceCancelledError) {
|
|
126
|
+
log.info(`downloadAsset cancelled requestId=${requestId}`);
|
|
127
|
+
throw error;
|
|
128
|
+
}
|
|
129
|
+
logger.error('Error downloading asset:', error);
|
|
130
|
+
return {
|
|
131
|
+
type: 'downloadAsset',
|
|
132
|
+
success: false,
|
|
133
|
+
error: error instanceof Error ? error.message : 'Unknown error'
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch (e_1) {
|
|
138
|
+
env_1.error = e_1;
|
|
139
|
+
env_1.hasError = true;
|
|
140
|
+
}
|
|
141
|
+
finally {
|
|
142
|
+
const result_1 = __disposeResources(env_1);
|
|
143
|
+
if (result_1)
|
|
144
|
+
await result_1;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { dispatchPluginReply } from '../handlers/plugin-dispatch.js';
|
|
2
|
+
import { startFinetune } from '../plugins/builtin/llamacpp-completion/ops/finetune.js';
|
|
3
|
+
export async function handleFinetune(request, progressCallback) {
|
|
4
|
+
if (progressCallback &&
|
|
5
|
+
(request.operation === 'start' ||
|
|
6
|
+
request.operation === 'resume' ||
|
|
7
|
+
request.operation === undefined)) {
|
|
8
|
+
return startFinetune(request, (progress) => {
|
|
9
|
+
const update = {
|
|
10
|
+
type: 'finetune:progress',
|
|
11
|
+
modelId: request.modelId,
|
|
12
|
+
...progress
|
|
13
|
+
};
|
|
14
|
+
progressCallback(update);
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
return dispatchPluginReply(request.modelId, 'finetune', request);
|
|
18
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { getModelEntry } from '../runtime/model-registry.js';
|
|
2
|
+
import { getPlugin } from '../plugins/registry.js';
|
|
3
|
+
import { detectToolDialectFromName } from '../utils/tools/index.js';
|
|
4
|
+
import { ModelNotFoundError } from '../errors/index.js';
|
|
5
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
6
|
+
const logger = getEngineLogger();
|
|
7
|
+
export function handleGetLoadedModelInfo(request) {
|
|
8
|
+
const { modelId } = request;
|
|
9
|
+
const entry = getModelEntry(modelId);
|
|
10
|
+
if (!entry) {
|
|
11
|
+
throw new ModelNotFoundError(modelId);
|
|
12
|
+
}
|
|
13
|
+
if (entry.isDelegated) {
|
|
14
|
+
const info = {
|
|
15
|
+
modelId: entry.id,
|
|
16
|
+
isDelegated: true,
|
|
17
|
+
handlers: [],
|
|
18
|
+
providerInfo: {
|
|
19
|
+
providerPublicKey: entry.delegated.providerPublicKey
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
return { type: 'getLoadedModelInfo', info };
|
|
23
|
+
}
|
|
24
|
+
const plugin = getPlugin(entry.local.modelType);
|
|
25
|
+
if (!plugin) {
|
|
26
|
+
logger.warn(`getLoadedModelInfo: no plugin registered for modelType "${entry.local.modelType}" on loaded model "${modelId}"`);
|
|
27
|
+
}
|
|
28
|
+
const handlers = plugin ? Object.keys(plugin.handlers) : [];
|
|
29
|
+
const info = {
|
|
30
|
+
modelId: entry.id,
|
|
31
|
+
isDelegated: false,
|
|
32
|
+
modelType: entry.local.modelType,
|
|
33
|
+
handlers,
|
|
34
|
+
loadedAt: entry.local.loadedAt,
|
|
35
|
+
...(plugin && { displayName: plugin.displayName }),
|
|
36
|
+
...(plugin && { addonPackage: plugin.addonPackage }),
|
|
37
|
+
...(entry.local.name && { name: entry.local.name }),
|
|
38
|
+
...(entry.local.path && { path: entry.local.path }),
|
|
39
|
+
// Same detection the completion normalizer uses, so callers see the dialect it parses.
|
|
40
|
+
...(handlers.includes('completionStream') && {
|
|
41
|
+
toolDialect: detectToolDialectFromName(entry.local.name, entry.local.path)
|
|
42
|
+
})
|
|
43
|
+
};
|
|
44
|
+
return { type: 'getLoadedModelInfo', info };
|
|
45
|
+
}
|