@qvac/inference 0.0.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +179 -0
- package/NOTICE +986 -0
- package/README.md +138 -0
- package/dist/api/audio-gen.d.ts +18 -0
- package/dist/api/audio-gen.js +148 -0
- package/dist/api/batch-completion.d.ts +14 -0
- package/dist/api/batch-completion.js +332 -0
- package/dist/api/bci-transcribe.d.ts +56 -0
- package/dist/api/bci-transcribe.js +172 -0
- package/dist/api/cancel.d.ts +34 -0
- package/dist/api/cancel.js +82 -0
- package/dist/api/classify.d.ts +22 -0
- package/dist/api/classify.js +46 -0
- package/dist/api/completion-stream.d.ts +106 -0
- package/dist/api/completion-stream.js +351 -0
- package/dist/api/delete-cache.d.ts +29 -0
- package/dist/api/delete-cache.js +49 -0
- package/dist/api/diffusion.d.ts +92 -0
- package/dist/api/diffusion.js +176 -0
- package/dist/api/download-asset.d.ts +49 -0
- package/dist/api/download-asset.js +87 -0
- package/dist/api/embed.d.ts +41 -0
- package/dist/api/embed.js +29 -0
- package/dist/api/finetune.d.ts +89 -0
- package/dist/api/finetune.js +138 -0
- package/dist/api/get-loaded-model-info.d.ts +24 -0
- package/dist/api/get-loaded-model-info.js +34 -0
- package/dist/api/get-model-info.d.ts +43 -0
- package/dist/api/get-model-info.js +21 -0
- package/dist/api/get-system-resources.d.ts +2 -0
- package/dist/api/get-system-resources.js +16 -0
- package/dist/api/heartbeat.d.ts +29 -0
- package/dist/api/heartbeat.js +37 -0
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +35 -0
- package/dist/api/invoke-plugin.d.ts +30 -0
- package/dist/api/invoke-plugin.js +54 -0
- package/dist/api/load-model.d.ts +169 -0
- package/dist/api/load-model.js +129 -0
- package/dist/api/logging-stream-registry.d.ts +4 -0
- package/dist/api/logging-stream-registry.js +61 -0
- package/dist/api/logging-stream.d.ts +23 -0
- package/dist/api/logging-stream.js +36 -0
- package/dist/api/ocr.d.ts +30 -0
- package/dist/api/ocr.js +88 -0
- package/dist/api/parse-input.d.ts +2 -0
- package/dist/api/parse-input.js +14 -0
- package/dist/api/provide.d.ts +23 -0
- package/dist/api/provide.js +32 -0
- package/dist/api/rag.d.ts +251 -0
- package/dist/api/rag.js +485 -0
- package/dist/api/registry.d.ts +39 -0
- package/dist/api/registry.js +63 -0
- package/dist/api/resume.d.ts +29 -0
- package/dist/api/resume.js +36 -0
- package/dist/api/state.d.ts +22 -0
- package/dist/api/state.js +29 -0
- package/dist/api/stop-provider.d.ts +20 -0
- package/dist/api/stop-provider.js +30 -0
- package/dist/api/subscribe-logs.d.ts +27 -0
- package/dist/api/subscribe-logs.js +43 -0
- package/dist/api/suspend.d.ts +30 -0
- package/dist/api/suspend.js +37 -0
- package/dist/api/text-to-speech.d.ts +64 -0
- package/dist/api/text-to-speech.js +434 -0
- package/dist/api/transcribe.d.ts +75 -0
- package/dist/api/transcribe.js +256 -0
- package/dist/api/translate.d.ts +47 -0
- package/dist/api/translate.js +115 -0
- package/dist/api/unload-model.d.ts +17 -0
- package/dist/api/unload-model.js +42 -0
- package/dist/api/upscale.d.ts +44 -0
- package/dist/api/upscale.js +99 -0
- package/dist/api/video.d.ts +86 -0
- package/dist/api/video.js +176 -0
- package/dist/api/vla-helpers.d.ts +43 -0
- package/dist/api/vla-helpers.js +124 -0
- package/dist/api/vla.d.ts +68 -0
- package/dist/api/vla.js +113 -0
- package/dist/config/config-utils.d.ts +4 -0
- package/dist/config/config-utils.js +18 -0
- package/dist/config/resolve-config.d.ts +8 -0
- package/dist/config/resolve-config.js +74 -0
- package/dist/constants/audio.d.ts +1 -0
- package/dist/constants/audio.js +1 -0
- package/dist/constants/index.d.ts +2 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/registry.d.ts +1 -0
- package/dist/constants/registry.js +1 -0
- package/dist/dispatch.d.ts +18 -0
- package/dist/dispatch.js +271 -0
- package/dist/engine.d.ts +5 -0
- package/dist/engine.js +27 -0
- package/dist/errors/index.d.ts +416 -0
- package/dist/errors/index.js +681 -0
- package/dist/errors/options.d.ts +2 -0
- package/dist/errors/options.js +13 -0
- package/dist/handlers/cancelHandler.d.ts +29 -0
- package/dist/handlers/cancelHandler.js +67 -0
- package/dist/handlers/completion-orchestrate.d.ts +41 -0
- package/dist/handlers/completion-orchestrate.js +190 -0
- package/dist/handlers/delete-cache.d.ts +11 -0
- package/dist/handlers/delete-cache.js +37 -0
- package/dist/handlers/download-asset.d.ts +2 -0
- package/dist/handlers/download-asset.js +146 -0
- package/dist/handlers/finetune.d.ts +2 -0
- package/dist/handlers/finetune.js +18 -0
- package/dist/handlers/get-loaded-model-info.d.ts +2 -0
- package/dist/handlers/get-loaded-model-info.js +45 -0
- package/dist/handlers/get-model-info.d.ts +2 -0
- package/dist/handlers/get-model-info.js +173 -0
- package/dist/handlers/get-system-resources.d.ts +2 -0
- package/dist/handlers/get-system-resources.js +53 -0
- package/dist/handlers/heartbeat.d.ts +2 -0
- package/dist/handlers/heartbeat.js +3 -0
- package/dist/handlers/load-model/download-manager.d.ts +95 -0
- package/dist/handlers/load-model/download-manager.js +306 -0
- package/dist/handlers/load-model/download-stats.d.ts +7 -0
- package/dist/handlers/load-model/download-stats.js +126 -0
- package/dist/handlers/load-model/handler.d.ts +2 -0
- package/dist/handlers/load-model/handler.js +287 -0
- package/dist/handlers/load-model/http.d.ts +3 -0
- package/dist/handlers/load-model/http.js +857 -0
- package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
- package/dist/handlers/load-model/hyperdrive.js +560 -0
- package/dist/handlers/load-model/index.d.ts +2 -0
- package/dist/handlers/load-model/index.js +2 -0
- package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/handlers/load-model/registry-client-options.js +18 -0
- package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/handlers/load-model/registry-companion-set.js +206 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
- package/dist/handlers/load-model/registry-download-utils.js +112 -0
- package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
- package/dist/handlers/load-model/registry-metadata.js +23 -0
- package/dist/handlers/load-model/registry.d.ts +4 -0
- package/dist/handlers/load-model/registry.js +298 -0
- package/dist/handlers/load-model/resolve-session.d.ts +33 -0
- package/dist/handlers/load-model/resolve-session.js +62 -0
- package/dist/handlers/load-model/resolve.d.ts +5 -0
- package/dist/handlers/load-model/resolve.js +158 -0
- package/dist/handlers/load-model/types.d.ts +59 -0
- package/dist/handlers/load-model/types.js +28 -0
- package/dist/handlers/logging-stream.d.ts +2 -0
- package/dist/handlers/logging-stream.js +39 -0
- package/dist/handlers/plugin-dispatch.d.ts +3 -0
- package/dist/handlers/plugin-dispatch.js +63 -0
- package/dist/handlers/plugin-invoke.d.ts +3 -0
- package/dist/handlers/plugin-invoke.js +81 -0
- package/dist/handlers/registry.d.ts +4 -0
- package/dist/handlers/registry.js +156 -0
- package/dist/handlers/resume.d.ts +2 -0
- package/dist/handlers/resume.js +11 -0
- package/dist/handlers/state.d.ts +2 -0
- package/dist/handlers/state.js +7 -0
- package/dist/handlers/suspend.d.ts +2 -0
- package/dist/handlers/suspend.js +11 -0
- package/dist/handlers/types.d.ts +13 -0
- package/dist/handlers/types.js +1 -0
- package/dist/handlers/unload-model.d.ts +2 -0
- package/dist/handlers/unload-model.js +29 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +12 -0
- package/dist/logging/addon.d.ts +5 -0
- package/dist/logging/addon.js +75 -0
- package/dist/logging/index.d.ts +5 -0
- package/dist/logging/index.js +4 -0
- package/dist/logging/logger.d.ts +41 -0
- package/dist/logging/logger.js +209 -0
- package/dist/logging/namespaces.d.ts +6 -0
- package/dist/logging/namespaces.js +7 -0
- package/dist/logging/types.d.ts +37 -0
- package/dist/logging/types.js +1 -0
- package/dist/logging/utils.d.ts +5 -0
- package/dist/logging/utils.js +113 -0
- package/dist/models/registry/index.d.ts +18 -0
- package/dist/models/registry/index.js +21 -0
- package/dist/models/registry/models.d.ts +22680 -0
- package/dist/models/registry/models.js +23908 -0
- package/dist/p2p/cancel-delegated.d.ts +3 -0
- package/dist/p2p/cancel-delegated.js +64 -0
- package/dist/p2p/completion-stream-delegated.d.ts +4 -0
- package/dist/p2p/completion-stream-delegated.js +51 -0
- package/dist/p2p/delegate-client.d.ts +9 -0
- package/dist/p2p/delegate-client.js +219 -0
- package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
- package/dist/p2p/delegate-connect-diagnostics.js +38 -0
- package/dist/p2p/delegate-transport.d.ts +18 -0
- package/dist/p2p/delegate-transport.js +237 -0
- package/dist/p2p/error-handlers.d.ts +4 -0
- package/dist/p2p/error-handlers.js +31 -0
- package/dist/p2p/handle-request.d.ts +2 -0
- package/dist/p2p/handle-request.js +171 -0
- package/dist/p2p/handler-utils.d.ts +19 -0
- package/dist/p2p/handler-utils.js +180 -0
- package/dist/p2p/heartbeat-delegated.d.ts +3 -0
- package/dist/p2p/heartbeat-delegated.js +32 -0
- package/dist/p2p/load-model-delegated.d.ts +6 -0
- package/dist/p2p/load-model-delegated.js +132 -0
- package/dist/p2p/progress-throttle.d.ts +7 -0
- package/dist/p2p/progress-throttle.js +47 -0
- package/dist/p2p/provideHandler/connection.d.ts +2 -0
- package/dist/p2p/provideHandler/connection.js +38 -0
- package/dist/p2p/provideHandler/handler.d.ts +2 -0
- package/dist/p2p/provideHandler/handler.js +66 -0
- package/dist/p2p/provideHandler/proxy.d.ts +2 -0
- package/dist/p2p/provideHandler/proxy.js +24 -0
- package/dist/p2p/provideHandler.d.ts +1 -0
- package/dist/p2p/provideHandler.js +2 -0
- package/dist/p2p/rpc-utils.d.ts +4 -0
- package/dist/p2p/rpc-utils.js +8 -0
- package/dist/p2p/stopProvideHandler.d.ts +2 -0
- package/dist/p2p/stopProvideHandler.js +29 -0
- package/dist/p2p/swarm.d.ts +9 -0
- package/dist/p2p/swarm.js +103 -0
- package/dist/p2p/transport-selector.d.ts +2 -0
- package/dist/p2p/transport-selector.js +18 -0
- package/dist/p2p/unload-model-delegated.d.ts +2 -0
- package/dist/p2p/unload-model-delegated.js +31 -0
- package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
- package/dist/plugins/builtin/asr-ggml/config.js +46 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
- package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
- package/dist/plugins/builtin/index.d.ts +11 -0
- package/dist/plugins/builtin/index.js +11 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
- package/dist/plugins/index.d.ts +9 -0
- package/dist/plugins/index.js +12 -0
- package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
- package/dist/plugins/ops/bci-transcribe.js +248 -0
- package/dist/plugins/ops/cancel.d.ts +25 -0
- package/dist/plugins/ops/cancel.js +43 -0
- package/dist/plugins/ops/embed.d.ts +15 -0
- package/dist/plugins/ops/embed.js +193 -0
- package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
- package/dist/plugins/ops/kv-cache-retention.js +121 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
- package/dist/plugins/ops/kv-cache-utils.js +120 -0
- package/dist/plugins/ops/load-model.d.ts +8 -0
- package/dist/plugins/ops/load-model.js +82 -0
- package/dist/plugins/ops/transcribe.d.ts +20 -0
- package/dist/plugins/ops/transcribe.js +343 -0
- package/dist/plugins/ops/translate.d.ts +7 -0
- package/dist/plugins/ops/translate.js +256 -0
- package/dist/plugins/ops/unload-model.d.ts +2 -0
- package/dist/plugins/ops/unload-model.js +45 -0
- package/dist/plugins/registry.d.ts +9 -0
- package/dist/plugins/registry.js +107 -0
- package/dist/profiling/aggregator.d.ts +14 -0
- package/dist/profiling/aggregator.js +100 -0
- package/dist/profiling/backend-diagnostics.d.ts +8 -0
- package/dist/profiling/backend-diagnostics.js +36 -0
- package/dist/profiling/clock.d.ts +8 -0
- package/dist/profiling/clock.js +23 -0
- package/dist/profiling/controller.d.ts +34 -0
- package/dist/profiling/controller.js +130 -0
- package/dist/profiling/delegation-profiler.d.ts +31 -0
- package/dist/profiling/delegation-profiler.js +155 -0
- package/dist/profiling/envelope.d.ts +10 -0
- package/dist/profiling/envelope.js +45 -0
- package/dist/profiling/events.d.ts +19 -0
- package/dist/profiling/events.js +35 -0
- package/dist/profiling/exporters.d.ts +9 -0
- package/dist/profiling/exporters.js +242 -0
- package/dist/profiling/index.d.ts +91 -0
- package/dist/profiling/index.js +87 -0
- package/dist/profiling/model-execution.d.ts +15 -0
- package/dist/profiling/model-execution.js +34 -0
- package/dist/profiling/operation-metrics.d.ts +24 -0
- package/dist/profiling/operation-metrics.js +370 -0
- package/dist/profiling/operation-wrappers.d.ts +17 -0
- package/dist/profiling/operation-wrappers.js +196 -0
- package/dist/profiling/profiler.d.ts +27 -0
- package/dist/profiling/profiler.js +127 -0
- package/dist/profiling/ring-buffer.d.ts +18 -0
- package/dist/profiling/ring-buffer.js +83 -0
- package/dist/profiling/types.d.ts +52 -0
- package/dist/profiling/types.js +1 -0
- package/dist/rag/handler.d.ts +2 -0
- package/dist/rag/handler.js +272 -0
- package/dist/rag/handlers/chunk.d.ts +2 -0
- package/dist/rag/handlers/chunk.js +7 -0
- package/dist/rag/handlers/close-workspace.d.ts +2 -0
- package/dist/rag/handlers/close-workspace.js +9 -0
- package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
- package/dist/rag/handlers/delete-embeddings.js +11 -0
- package/dist/rag/handlers/delete-workspace.d.ts +2 -0
- package/dist/rag/handlers/delete-workspace.js +14 -0
- package/dist/rag/handlers/ingest.d.ts +9 -0
- package/dist/rag/handlers/ingest.js +19 -0
- package/dist/rag/handlers/list-workspaces.d.ts +2 -0
- package/dist/rag/handlers/list-workspaces.js +4 -0
- package/dist/rag/handlers/reindex.d.ts +9 -0
- package/dist/rag/handlers/reindex.js +16 -0
- package/dist/rag/handlers/save-embeddings.d.ts +9 -0
- package/dist/rag/handlers/save-embeddings.js +15 -0
- package/dist/rag/handlers/search.d.ts +2 -0
- package/dist/rag/handlers/search.js +16 -0
- package/dist/rag/index.d.ts +11 -0
- package/dist/rag/index.js +11 -0
- package/dist/rag/rag-operation-manager.d.ts +28 -0
- package/dist/rag/rag-operation-manager.js +76 -0
- package/dist/rag/rag-workspace-manager.d.ts +14 -0
- package/dist/rag/rag-workspace-manager.js +143 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +166 -0
- package/dist/resources/collector.d.ts +643 -0
- package/dist/resources/collector.js +226 -0
- package/dist/resources/instance.d.ts +4 -0
- package/dist/resources/instance.js +15 -0
- package/dist/resources/native.d.ts +2 -0
- package/dist/resources/native.js +35 -0
- package/dist/resources/normalize.d.ts +12 -0
- package/dist/resources/normalize.js +73 -0
- package/dist/resources/types.d.ts +65 -0
- package/dist/resources/types.js +1 -0
- package/dist/runtime/cache-lock.d.ts +2 -0
- package/dist/runtime/cache-lock.js +67 -0
- package/dist/runtime/disposable-scope.d.ts +35 -0
- package/dist/runtime/disposable-scope.js +69 -0
- package/dist/runtime/env.d.ts +16 -0
- package/dist/runtime/env.js +53 -0
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +20 -0
- package/dist/runtime/lifecycle.js +102 -0
- package/dist/runtime/logging-stream-registry.d.ts +27 -0
- package/dist/runtime/logging-stream-registry.js +133 -0
- package/dist/runtime/model-config-utils.d.ts +14 -0
- package/dist/runtime/model-config-utils.js +128 -0
- package/dist/runtime/model-registry.d.ts +74 -0
- package/dist/runtime/model-registry.js +130 -0
- package/dist/runtime/registry-client.d.ts +4 -0
- package/dist/runtime/registry-client.js +118 -0
- package/dist/runtime/request-context.d.ts +72 -0
- package/dist/runtime/request-context.js +114 -0
- package/dist/runtime/request-id.d.ts +17 -0
- package/dist/runtime/request-id.js +29 -0
- package/dist/runtime/request-registry.d.ts +194 -0
- package/dist/runtime/request-registry.js +546 -0
- package/dist/runtime/runtime-lifecycle.d.ts +37 -0
- package/dist/runtime/runtime-lifecycle.js +163 -0
- package/dist/runtime/state.d.ts +26 -0
- package/dist/runtime/state.js +138 -0
- package/dist/schemas/audio-gen.d.ts +252 -0
- package/dist/schemas/audio-gen.js +69 -0
- package/dist/schemas/batch-completion-stream.d.ts +467 -0
- package/dist/schemas/batch-completion-stream.js +123 -0
- package/dist/schemas/bci-config.d.ts +74 -0
- package/dist/schemas/bci-config.js +58 -0
- package/dist/schemas/bci.d.ts +178 -0
- package/dist/schemas/bci.js +89 -0
- package/dist/schemas/cancel.d.ts +203 -0
- package/dist/schemas/cancel.js +135 -0
- package/dist/schemas/classification.d.ts +52 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +6420 -0
- package/dist/schemas/common.js +133 -0
- package/dist/schemas/completion-event.d.ts +248 -0
- package/dist/schemas/completion-event.js +94 -0
- package/dist/schemas/completion-stream.d.ts +475 -0
- package/dist/schemas/completion-stream.js +236 -0
- package/dist/schemas/config.d.ts +5022 -0
- package/dist/schemas/config.js +169 -0
- package/dist/schemas/delegate.d.ts +29 -0
- package/dist/schemas/delegate.js +40 -0
- package/dist/schemas/download-asset.d.ts +225 -0
- package/dist/schemas/download-asset.js +71 -0
- package/dist/schemas/embed.d.ts +41 -0
- package/dist/schemas/embed.js +35 -0
- package/dist/schemas/engine-addon-map.d.ts +20 -0
- package/dist/schemas/engine-addon-map.js +78 -0
- package/dist/schemas/error.d.ts +40 -0
- package/dist/schemas/error.js +65 -0
- package/dist/schemas/errors.d.ts +119 -0
- package/dist/schemas/errors.js +615 -0
- package/dist/schemas/finetune.d.ts +621 -0
- package/dist/schemas/finetune.js +163 -0
- package/dist/schemas/index.d.ts +45 -0
- package/dist/schemas/index.js +45 -0
- package/dist/schemas/json-schema.d.ts +15 -0
- package/dist/schemas/json-schema.js +2 -0
- package/dist/schemas/lifecycle.d.ts +38 -0
- package/dist/schemas/lifecycle.js +21 -0
- package/dist/schemas/llamacpp-config.d.ts +403 -0
- package/dist/schemas/llamacpp-config.js +133 -0
- package/dist/schemas/load-model.d.ts +25635 -0
- package/dist/schemas/load-model.js +629 -0
- package/dist/schemas/logging-stream.d.ts +33 -0
- package/dist/schemas/logging-stream.js +17 -0
- package/dist/schemas/mcp-adapter.d.ts +46 -0
- package/dist/schemas/mcp-adapter.js +1 -0
- package/dist/schemas/model-file.d.ts +27 -0
- package/dist/schemas/model-file.js +29 -0
- package/dist/schemas/model-info.d.ts +197 -0
- package/dist/schemas/model-info.js +144 -0
- package/dist/schemas/model-ops.d.ts +36 -0
- package/dist/schemas/model-ops.js +40 -0
- package/dist/schemas/model-src-utils.d.ts +192 -0
- package/dist/schemas/model-src-utils.js +61 -0
- package/dist/schemas/model-types.d.ts +275 -0
- package/dist/schemas/model-types.js +223 -0
- package/dist/schemas/ocr.d.ts +138 -0
- package/dist/schemas/ocr.js +59 -0
- package/dist/schemas/path-validation.d.ts +9 -0
- package/dist/schemas/path-validation.js +22 -0
- package/dist/schemas/plugin.d.ts +341 -0
- package/dist/schemas/plugin.js +241 -0
- package/dist/schemas/profiling.d.ts +695 -0
- package/dist/schemas/profiling.js +100 -0
- package/dist/schemas/provide.d.ts +50 -0
- package/dist/schemas/provide.js +32 -0
- package/dist/schemas/rag.d.ts +301 -0
- package/dist/schemas/rag.js +211 -0
- package/dist/schemas/registry.d.ts +453 -0
- package/dist/schemas/registry.js +104 -0
- package/dist/schemas/reload-config.d.ts +585 -0
- package/dist/schemas/reload-config.js +42 -0
- package/dist/schemas/runtime-context.d.ts +18 -0
- package/dist/schemas/runtime-context.js +7 -0
- package/dist/schemas/sdcpp-config.d.ts +1126 -0
- package/dist/schemas/sdcpp-config.js +748 -0
- package/dist/schemas/system-resources.d.ts +3966 -0
- package/dist/schemas/system-resources.js +149 -0
- package/dist/schemas/text-to-speech.d.ts +1352 -0
- package/dist/schemas/text-to-speech.js +364 -0
- package/dist/schemas/tools.d.ts +80 -0
- package/dist/schemas/tools.js +52 -0
- package/dist/schemas/transcription-config.d.ts +161 -0
- package/dist/schemas/transcription-config.js +156 -0
- package/dist/schemas/transcription.d.ts +261 -0
- package/dist/schemas/transcription.js +145 -0
- package/dist/schemas/translate.d.ts +138 -0
- package/dist/schemas/translate.js +103 -0
- package/dist/schemas/translation-config.d.ts +1031 -0
- package/dist/schemas/translation-config.js +170 -0
- package/dist/schemas/vla.d.ts +110 -0
- package/dist/schemas/vla.js +151 -0
- package/dist/selection.d.ts +8 -0
- package/dist/selection.js +14 -0
- package/dist/surface.d.ts +18 -0
- package/dist/surface.js +30 -0
- package/dist/utils/addon-responses.d.ts +58 -0
- package/dist/utils/addon-responses.js +1 -0
- package/dist/utils/aggregate-events.d.ts +25 -0
- package/dist/utils/aggregate-events.js +73 -0
- package/dist/utils/archive.d.ts +4 -0
- package/dist/utils/archive.js +180 -0
- package/dist/utils/asr-events.d.ts +14 -0
- package/dist/utils/asr-events.js +30 -0
- package/dist/utils/audio/decoder.d.ts +4 -0
- package/dist/utils/audio/decoder.js +80 -0
- package/dist/utils/audio-input.d.ts +9 -0
- package/dist/utils/audio-input.js +34 -0
- package/dist/utils/cache/auto-history.d.ts +3 -0
- package/dist/utils/cache/auto-history.js +26 -0
- package/dist/utils/cache/paths.d.ts +42 -0
- package/dist/utils/cache/paths.js +133 -0
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/cache/types.js +1 -0
- package/dist/utils/cache-normalize.d.ts +9 -0
- package/dist/utils/cache-normalize.js +20 -0
- package/dist/utils/checksum.d.ts +3 -0
- package/dist/utils/checksum.js +29 -0
- package/dist/utils/clear-storage.d.ts +4 -0
- package/dist/utils/clear-storage.js +14 -0
- package/dist/utils/completion-normalizer.d.ts +17 -0
- package/dist/utils/completion-normalizer.js +453 -0
- package/dist/utils/config-transform.d.ts +2 -0
- package/dist/utils/config-transform.js +21 -0
- package/dist/utils/decorate-promise.d.ts +33 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/encoding.d.ts +16 -0
- package/dist/utils/encoding.js +27 -0
- package/dist/utils/expand-gguf-shards.d.ts +2 -0
- package/dist/utils/expand-gguf-shards.js +22 -0
- package/dist/utils/formatting.d.ts +18 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
- package/dist/utils/gguf-tensor-extractor.js +311 -0
- package/dist/utils/global-singleton.d.ts +1 -0
- package/dist/utils/global-singleton.js +10 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/load-model-validation.d.ts +6 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/dist/utils/mcp-adapter.d.ts +11 -0
- package/dist/utils/mcp-adapter.js +96 -0
- package/dist/utils/multi-gpu-mobile.d.ts +4 -0
- package/dist/utils/multi-gpu-mobile.js +10 -0
- package/dist/utils/object.d.ts +1 -0
- package/dist/utils/object.js +3 -0
- package/dist/utils/path-security.d.ts +22 -0
- package/dist/utils/path-security.js +69 -0
- package/dist/utils/qvac-paths.d.ts +1 -0
- package/dist/utils/qvac-paths.js +5 -0
- package/dist/utils/response-format.d.ts +2 -0
- package/dist/utils/response-format.js +20 -0
- package/dist/utils/shard-utils.d.ts +47 -0
- package/dist/utils/shard-utils.js +123 -0
- package/dist/utils/tool-helpers.d.ts +24 -0
- package/dist/utils/tool-helpers.js +111 -0
- package/dist/utils/tool-integration.d.ts +23 -0
- package/dist/utils/tool-integration.js +29 -0
- package/dist/utils/tools/dialect.d.ts +7 -0
- package/dist/utils/tools/dialect.js +20 -0
- package/dist/utils/tools/index.d.ts +2 -0
- package/dist/utils/tools/index.js +2 -0
- package/dist/utils/tools/parser.d.ts +5 -0
- package/dist/utils/tools/parser.js +69 -0
- package/dist/utils/tools/parsers/dsml.d.ts +3 -0
- package/dist/utils/tools/parsers/dsml.js +177 -0
- package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
- package/dist/utils/tools/parsers/gemma4native.js +65 -0
- package/dist/utils/tools/parsers/harmony.d.ts +3 -0
- package/dist/utils/tools/parsers/harmony.js +57 -0
- package/dist/utils/tools/parsers/hermes.d.ts +3 -0
- package/dist/utils/tools/parsers/hermes.js +122 -0
- package/dist/utils/tools/parsers/json.d.ts +5 -0
- package/dist/utils/tools/parsers/json.js +115 -0
- package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
- package/dist/utils/tools/parsers/pythonic.js +267 -0
- package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
- package/dist/utils/tools/parsers/qwen35.js +125 -0
- package/dist/utils/tools/shared.d.ts +17 -0
- package/dist/utils/tools/shared.js +52 -0
- package/dist/utils/transcribe-metadata.d.ts +20 -0
- package/dist/utils/transcribe-metadata.js +28 -0
- package/dist/utils/tts-stats.d.ts +25 -0
- package/dist/utils/tts-stats.js +16 -0
- package/dist/utils/withTimeout.d.ts +17 -0
- package/dist/utils/withTimeout.js +47 -0
- package/dist/utils/zod-error.d.ts +2 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +266 -1
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { CompletionParams, CompletionStats, GenerationParams, ResponseFormat, Tool, ToolCall, ToolDialect } from '../../../../schemas/index.js';
|
|
3
|
+
import type { DisposableScope } from '../../../../runtime/disposable-scope.js';
|
|
4
|
+
import type { Logger } from '../../../../logging/types.js';
|
|
5
|
+
interface CompletionResult {
|
|
6
|
+
modelExecutionMs: number;
|
|
7
|
+
stats?: CompletionStats;
|
|
8
|
+
toolCalls: ToolCall[];
|
|
9
|
+
}
|
|
10
|
+
interface ChatHistory {
|
|
11
|
+
role?: string;
|
|
12
|
+
content?: string;
|
|
13
|
+
type?: string;
|
|
14
|
+
name?: string;
|
|
15
|
+
description?: string;
|
|
16
|
+
parameters?: unknown;
|
|
17
|
+
}
|
|
18
|
+
export type CompletionGenerationParams = GenerationParams & {
|
|
19
|
+
json_schema?: string;
|
|
20
|
+
};
|
|
21
|
+
export declare function transformMessages(messages: Array<{
|
|
22
|
+
role: string;
|
|
23
|
+
content: string;
|
|
24
|
+
attachments?: {
|
|
25
|
+
path: string;
|
|
26
|
+
}[] | undefined;
|
|
27
|
+
} | Tool>): ChatHistory[];
|
|
28
|
+
export declare function completion(params: CompletionParams & {
|
|
29
|
+
tools?: Tool[];
|
|
30
|
+
generationParams?: GenerationParams;
|
|
31
|
+
toolDialect?: ToolDialect;
|
|
32
|
+
responseFormat?: ResponseFormat;
|
|
33
|
+
}, opts: {
|
|
34
|
+
signal: AbortSignal;
|
|
35
|
+
scope: DisposableScope;
|
|
36
|
+
/**
|
|
37
|
+
* Request-scoped logger forwarded to `createKvCacheSession` so
|
|
38
|
+
* kv-cache lines share the request's lifecycle prefix. Falls
|
|
39
|
+
* back to the module-level server logger when omitted.
|
|
40
|
+
*/
|
|
41
|
+
logger?: Logger;
|
|
42
|
+
}): AsyncGenerator<{
|
|
43
|
+
token: string;
|
|
44
|
+
}, CompletionResult, unknown>;
|
|
45
|
+
export {};
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
import { TOOLS_MODE } from '../../../../schemas/tools.js';
|
|
2
|
+
import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
|
|
3
|
+
import { extractSystemPrompt, getCurrentCacheInfo } from '../../../../plugins/ops/kv-cache-utils.js';
|
|
4
|
+
import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
|
|
5
|
+
import { decideCachedHistorySlice, shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
|
|
6
|
+
import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
|
|
7
|
+
import { appendToolsToHistory, detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
8
|
+
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
9
|
+
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
10
|
+
import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
|
|
11
|
+
import { getEngineLogger } from '../../../../logging/index.js';
|
|
12
|
+
import { AttachmentNotFoundError } from '../../../../errors/index.js';
|
|
13
|
+
import { nowMs } from '../../../../profiling/index.js';
|
|
14
|
+
import { buildStreamResult } from '../../../../profiling/model-execution.js';
|
|
15
|
+
import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
|
|
16
|
+
import fs from 'bare-fs';
|
|
17
|
+
const logger = getEngineLogger();
|
|
18
|
+
function transformMessage(message) {
|
|
19
|
+
const transformed = [];
|
|
20
|
+
// Check if it's a tool definition (has type: "function")
|
|
21
|
+
if ('type' in message && message.type === 'function') {
|
|
22
|
+
transformed.push({
|
|
23
|
+
type: 'function',
|
|
24
|
+
name: message.name,
|
|
25
|
+
description: message.description,
|
|
26
|
+
parameters: message.parameters
|
|
27
|
+
});
|
|
28
|
+
return transformed;
|
|
29
|
+
}
|
|
30
|
+
const msg = message;
|
|
31
|
+
if (msg.attachments && msg.attachments.length > 0) {
|
|
32
|
+
for (const attachment of msg.attachments) {
|
|
33
|
+
if (!fs.existsSync(attachment.path)) {
|
|
34
|
+
throw new AttachmentNotFoundError(attachment.path);
|
|
35
|
+
}
|
|
36
|
+
transformed.push({
|
|
37
|
+
role: msg.role,
|
|
38
|
+
content: attachment.path,
|
|
39
|
+
type: 'media'
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
transformed.push({
|
|
44
|
+
role: msg.role,
|
|
45
|
+
content: msg.content
|
|
46
|
+
});
|
|
47
|
+
return transformed;
|
|
48
|
+
}
|
|
49
|
+
function runModel(model, prompt, opts) {
|
|
50
|
+
return model.run(prompt, opts);
|
|
51
|
+
}
|
|
52
|
+
export function transformMessages(messages) {
|
|
53
|
+
const transformed = [];
|
|
54
|
+
for (const message of messages) {
|
|
55
|
+
transformed.push(...transformMessage(message));
|
|
56
|
+
}
|
|
57
|
+
return transformed;
|
|
58
|
+
}
|
|
59
|
+
async function initSystemPromptCache(model, cachePathToUse, systemPromptToUse, cacheKey, tools) {
|
|
60
|
+
const primeMessages = [{ role: 'system', content: systemPromptToUse }];
|
|
61
|
+
let toolCount = 0;
|
|
62
|
+
if (tools && tools.length > 0) {
|
|
63
|
+
const transformedTools = transformMessages(tools);
|
|
64
|
+
primeMessages.push(...transformedTools);
|
|
65
|
+
toolCount = tools.length;
|
|
66
|
+
}
|
|
67
|
+
logCacheInit(cacheKey, systemPromptToUse, toolCount);
|
|
68
|
+
logMessagesToAddon(primeMessages, 'CACHE_INIT');
|
|
69
|
+
const primeResponse = await runModel(model, primeMessages, {
|
|
70
|
+
cacheKey: cachePathToUse,
|
|
71
|
+
saveCacheToDisk: true,
|
|
72
|
+
prefill: true
|
|
73
|
+
});
|
|
74
|
+
await primeResponse.await();
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Pick the messages that need to reach the model for the next turn.
|
|
78
|
+
*
|
|
79
|
+
* Static mode (no `tools` argument):
|
|
80
|
+
* - Cache miss: send the whole history minus the system message (which
|
|
81
|
+
* was primed during cache init).
|
|
82
|
+
* - Cache hit with a recorded `savedCount`: send only the unsaved tail
|
|
83
|
+
* (`history.slice(savedCount)`), so a multi-message turn (e.g. a
|
|
84
|
+
* consumer pushing both an assistant transcript and a follow-up user
|
|
85
|
+
* message between completions) all reaches the model.
|
|
86
|
+
* - Cache hit with a stale/missing `savedCount`: fall back to the full
|
|
87
|
+
* non-system history. The session is told (`dropStaleSavedCount`) so
|
|
88
|
+
* the bad boundary doesn't propagate into the next turn.
|
|
89
|
+
*
|
|
90
|
+
* Dynamic mode (`tools` argument set):
|
|
91
|
+
* - The addon anchors the tool block after the last user message and
|
|
92
|
+
* trims tools + the assistant's tool-call output from the cache once
|
|
93
|
+
* the chain resolves. After that trim, the cache only holds messages
|
|
94
|
+
* up to the last user turn, so we ship the right slice
|
|
95
|
+
* plus the (possibly new) tool set:
|
|
96
|
+
* * tool-chain continuation (last role is "tool"): send the trailing
|
|
97
|
+
* consecutive tool messages, no tool block — tools are still
|
|
98
|
+
* anchored in the cache from the previous round.
|
|
99
|
+
* * new user turn after a chain (prev role is "assistant"): send
|
|
100
|
+
* [assistant, user] so the model sees its own final reply before
|
|
101
|
+
* the new prompt, then re-anchor the tool block.
|
|
102
|
+
* * otherwise: send just the last message + tool block.
|
|
103
|
+
*/
|
|
104
|
+
function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
|
|
105
|
+
const addTools = tools?.length ? transformMessages(tools) : [];
|
|
106
|
+
const dynamic = addTools.length > 0;
|
|
107
|
+
if (!(cacheExists && history.length > 0)) {
|
|
108
|
+
const historyWithoutSystem = history.filter((msg) => msg.role !== 'system');
|
|
109
|
+
return [...transformMessages(historyWithoutSystem), ...addTools];
|
|
110
|
+
}
|
|
111
|
+
if (!dynamic) {
|
|
112
|
+
// Static path — slice from the turn's `savedCount` so callers can
|
|
113
|
+
// stage multiple messages between completions. `decideCachedHistorySlice`
|
|
114
|
+
// also guards against the QVAC-17780 stale-count regression: if the
|
|
115
|
+
// saved boundary would slice the history down to an empty payload
|
|
116
|
+
// (e.g. after a cancelled mid-decode), it falls back to the full
|
|
117
|
+
// non-system history and signals the caller to drop the bad entry.
|
|
118
|
+
// The session owns the entry; `dropStaleSavedCount` clears it
|
|
119
|
+
// without touching the on-disk file (the file is still trustworthy
|
|
120
|
+
// — only the boundary count is wrong).
|
|
121
|
+
const { messages, clearStaleCount } = decideCachedHistorySlice(turn.savedCount, cacheExists, history);
|
|
122
|
+
if (clearStaleCount) {
|
|
123
|
+
session.dropStaleSavedCount(turn);
|
|
124
|
+
}
|
|
125
|
+
return transformMessages(messages);
|
|
126
|
+
}
|
|
127
|
+
// Dynamic path. The addon trimmed tools after the previous round, so the
|
|
128
|
+
// cache no longer holds the saved-count we'd rely on for slicing — pick
|
|
129
|
+
// the right fragment based on the role of the last history message.
|
|
130
|
+
const lastMsg = history[history.length - 1];
|
|
131
|
+
if (lastMsg.role === 'tool') {
|
|
132
|
+
const trailingTools = [];
|
|
133
|
+
for (let i = history.length - 1; i >= 0; i--) {
|
|
134
|
+
const msg = history[i];
|
|
135
|
+
if (msg.role !== 'tool')
|
|
136
|
+
break;
|
|
137
|
+
trailingTools.unshift(msg);
|
|
138
|
+
}
|
|
139
|
+
return transformMessages(trailingTools);
|
|
140
|
+
}
|
|
141
|
+
if (lastMsg.role === 'user') {
|
|
142
|
+
const prevMsg = history[history.length - 2];
|
|
143
|
+
const tail = prevMsg?.role === 'assistant' ? [prevMsg, lastMsg] : [lastMsg];
|
|
144
|
+
return [...transformMessages(tail), ...addTools];
|
|
145
|
+
}
|
|
146
|
+
return [...transformMessages([lastMsg]), ...addTools];
|
|
147
|
+
}
|
|
148
|
+
async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect) {
|
|
149
|
+
const runOptions = {
|
|
150
|
+
...(generationParams && { generationParams }),
|
|
151
|
+
...(cacheOptions?.cacheKey !== undefined && {
|
|
152
|
+
cacheKey: cacheOptions.cacheKey
|
|
153
|
+
}),
|
|
154
|
+
...(cacheOptions?.saveCacheToDisk !== undefined && {
|
|
155
|
+
saveCacheToDisk: cacheOptions.saveCacheToDisk
|
|
156
|
+
})
|
|
157
|
+
};
|
|
158
|
+
const hasRunOptions = Object.keys(runOptions).length > 0;
|
|
159
|
+
const modelStart = nowMs();
|
|
160
|
+
const response = await runModel(model, messagesToSend, hasRunOptions ? runOptions : undefined);
|
|
161
|
+
let accumulatedText = '';
|
|
162
|
+
let producedTokens = false;
|
|
163
|
+
let emittedPieces = 0;
|
|
164
|
+
let toolCallsResult = [];
|
|
165
|
+
for await (const token of response.iterate()) {
|
|
166
|
+
const tokenStr = token;
|
|
167
|
+
if (tokenStr.length > 0) {
|
|
168
|
+
producedTokens = true;
|
|
169
|
+
emittedPieces++;
|
|
170
|
+
}
|
|
171
|
+
accumulatedText += tokenStr;
|
|
172
|
+
yield { token: tokenStr };
|
|
173
|
+
}
|
|
174
|
+
const modelExecutionMs = nowMs() - modelStart;
|
|
175
|
+
if (cacheOptions?.saveCacheToDisk && cacheOptions.cacheKey) {
|
|
176
|
+
logCacheSave(cacheOptions.cacheKey);
|
|
177
|
+
}
|
|
178
|
+
if (tools && tools.length > 0) {
|
|
179
|
+
const { toolCalls } = parseToolCalls(accumulatedText, tools, dialect);
|
|
180
|
+
toolCallsResult = toolCalls;
|
|
181
|
+
}
|
|
182
|
+
const responseWithStats = response;
|
|
183
|
+
const stats = withEmittedTokens(normalizeCompletionStats(responseWithStats.stats), emittedPieces);
|
|
184
|
+
return {
|
|
185
|
+
...buildStreamResult(modelExecutionMs, stats),
|
|
186
|
+
toolCalls: toolCallsResult,
|
|
187
|
+
responseText: accumulatedText,
|
|
188
|
+
producedTokens
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
export async function* completion(params, opts) {
|
|
192
|
+
const { history, modelId, kvCache, tools, generationParams, responseFormat } = params;
|
|
193
|
+
const { signal, scope } = opts;
|
|
194
|
+
const requestLogger = opts.logger ?? logger;
|
|
195
|
+
const modelConfig = getModelConfig(modelId);
|
|
196
|
+
const toolsEnabled = modelConfig.tools === true;
|
|
197
|
+
const toolsMode = modelConfig.toolsMode;
|
|
198
|
+
const dynamicTools = !!tools?.length && toolsEnabled && toolsMode === TOOLS_MODE.dynamic;
|
|
199
|
+
const staticTools = !!tools?.length && toolsEnabled && !dynamicTools;
|
|
200
|
+
const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
|
|
201
|
+
// `responseFormat` is forwarded to the addon as a per-request
|
|
202
|
+
// `generationParams.json_schema`, which the addon converts to GBNF and
|
|
203
|
+
// applies for the duration of the request only. This avoids mutating
|
|
204
|
+
// the shared `modelConfig` and is therefore safe under concurrent
|
|
205
|
+
// completions on the same model. `tools` still constrain output through
|
|
206
|
+
// their parameter schema and the dialect-specific parser chain (mutually
|
|
207
|
+
// exclusive with a non-text `responseFormat` at the schema layer).
|
|
208
|
+
let mergedGenerationParams = generationParams;
|
|
209
|
+
if (responseFormat && !(tools && tools.length > 0)) {
|
|
210
|
+
const jsonSchema = getResponseFormatJsonSchema(responseFormat);
|
|
211
|
+
if (jsonSchema !== undefined) {
|
|
212
|
+
mergedGenerationParams = {
|
|
213
|
+
...(generationParams ?? {}),
|
|
214
|
+
json_schema: jsonSchema
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
const model = getModel(modelId);
|
|
219
|
+
// Hard-cancel wiring: when the registry aborts the request's signal,
|
|
220
|
+
// forward to the addon so the C++ work stops as soon as it can. The
|
|
221
|
+
// we still treat `signal.aborted` as the truth for cancel detection
|
|
222
|
+
// (post-completion bookkeeping below) — this listener only shortens
|
|
223
|
+
// the latency between "user clicked stop" and "addon stops decoding".
|
|
224
|
+
//
|
|
225
|
+
// Fire-and-forget by construction (event listeners can't `await`), but
|
|
226
|
+
// `addon.cancel()` returns a Promise — if it ever rejects the bare
|
|
227
|
+
// `void` would leak it as an unhandledRejection. Attach `.catch(...)`
|
|
228
|
+
// so a rejection is logged and the process stays clean; the iterator
|
|
229
|
+
// below still sees EOF/empty tokens via the addon's normal cancel path
|
|
230
|
+
// so callers aren't affected.
|
|
231
|
+
const onAbort = () => {
|
|
232
|
+
const addon = model.addon;
|
|
233
|
+
if (addon?.cancel) {
|
|
234
|
+
addon.cancel.call(addon).catch((err) => {
|
|
235
|
+
requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
};
|
|
239
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
240
|
+
// `addEventListener("abort", ..., { once: true })` does *not* fire if
|
|
241
|
+
// the signal is already aborted at register time — but the registry
|
|
242
|
+
// synchronously aborts a fresh controller when `parentSignal` was
|
|
243
|
+
// already aborted at `begin(...)`. Without this fall-through, the
|
|
244
|
+
// addon would keep decoding until the post-loop check notices.
|
|
245
|
+
// Re-using `onAbort` here keeps the listener body as the single
|
|
246
|
+
// source of truth for "what cancel does."
|
|
247
|
+
if (signal.aborted)
|
|
248
|
+
onAbort();
|
|
249
|
+
// Detach the abort listener on every exit path (happy, throw, generator
|
|
250
|
+
// `return()` from upstream). `{ once: true }` already removes the
|
|
251
|
+
// listener if the signal fires, so the `removeEventListener` here is
|
|
252
|
+
// the cleanup hook for the signal-never-fired path.
|
|
253
|
+
scope.defer(() => {
|
|
254
|
+
signal.removeEventListener('abort', onAbort);
|
|
255
|
+
});
|
|
256
|
+
if (!kvCache) {
|
|
257
|
+
// KV-cache disabled — straight passthrough, no session involvement.
|
|
258
|
+
let historyWithTools = history;
|
|
259
|
+
if (staticTools && tools) {
|
|
260
|
+
historyWithTools = prependToolsToHistory(history, tools);
|
|
261
|
+
}
|
|
262
|
+
else if (dynamicTools && tools) {
|
|
263
|
+
historyWithTools = appendToolsToHistory(history, tools);
|
|
264
|
+
}
|
|
265
|
+
const transformedHistory = transformMessages(historyWithTools);
|
|
266
|
+
logCacheDisabled();
|
|
267
|
+
logMessagesToAddon(transformedHistory, 'NO_CACHE');
|
|
268
|
+
return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect);
|
|
269
|
+
}
|
|
270
|
+
// ---- KV-cache path. The session owns all three bookkeeping layers
|
|
271
|
+
// (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`). The
|
|
272
|
+
// handler asks for a turn, registers rollback on the scope, and on
|
|
273
|
+
// the happy path calls `commitTurn` which short-circuits the deferred
|
|
274
|
+
// rollback. Cancellations / zero-token replies / rename failures all
|
|
275
|
+
// unwind through the same `scope.defer` hook. ----
|
|
276
|
+
const session = createKvCacheSession(modelId, { logger: requestLogger });
|
|
277
|
+
const systemPromptFromHistory = extractSystemPrompt(history);
|
|
278
|
+
// Dynamic mode lets each turn carry its own tool set, so the cache
|
|
279
|
+
// hash must not depend on the tool list — otherwise a tool change
|
|
280
|
+
// would force a fresh cache file and defeat the whole optimisation.
|
|
281
|
+
const configHash = generateConfigHash(systemPromptFromHistory, dynamicTools ? undefined : tools);
|
|
282
|
+
const systemPromptToUse = systemPromptFromHistory ||
|
|
283
|
+
modelConfig.system_prompt ||
|
|
284
|
+
'You are a helpful assistant.';
|
|
285
|
+
const primeIfMissing = async (cachePath) => {
|
|
286
|
+
await initSystemPromptCache(model, cachePath, systemPromptToUse, typeof kvCache === 'string' ? kvCache : 'auto',
|
|
287
|
+
// Static-mode tools are baked into the system-prompt cache so
|
|
288
|
+
// they're shared across the session. Dynamic-mode tools belong
|
|
289
|
+
// to a per-turn anchor and must not enter the system cache.
|
|
290
|
+
staticTools ? tools : undefined);
|
|
291
|
+
};
|
|
292
|
+
let turn;
|
|
293
|
+
if (typeof kvCache === 'string') {
|
|
294
|
+
turn = await session.beginTurn({
|
|
295
|
+
kind: 'custom',
|
|
296
|
+
customKey: kvCache,
|
|
297
|
+
configHash,
|
|
298
|
+
primeIfMissing
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
else {
|
|
302
|
+
const cacheMessages = history.map((msg) => ({
|
|
303
|
+
role: msg.role,
|
|
304
|
+
content: msg.content,
|
|
305
|
+
attachments: msg.attachments ?? undefined
|
|
306
|
+
}));
|
|
307
|
+
turn = await session.beginTurn({
|
|
308
|
+
kind: 'auto',
|
|
309
|
+
configHash,
|
|
310
|
+
history: cacheMessages,
|
|
311
|
+
primeIfMissing
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
// Single cleanup hook for every non-success exit path. `commitTurn`
|
|
315
|
+
// flips the turn's internal `committed` flag so this becomes a no-op
|
|
316
|
+
// on the happy path. Scope unwinding is LIFO — registered after the
|
|
317
|
+
// `removeEventListener` defer above so rollback runs before the
|
|
318
|
+
// listener detach.
|
|
319
|
+
scope.defer(() => session.rollback(turn));
|
|
320
|
+
// `cacheExists` is implied by `beginTurn` — the session either found
|
|
321
|
+
// an existing cache or just primed one. Pass `true` to the message
|
|
322
|
+
// selector so the slicing branches engage.
|
|
323
|
+
const messagesToSend = prepareMessagesForCache(session, turn,
|
|
324
|
+
/* cacheExists */ true, history, dynamicTools ? tools : undefined);
|
|
325
|
+
logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
|
|
326
|
+
const result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect);
|
|
327
|
+
const shouldCommitTurn = shouldCommitCachedTurn({
|
|
328
|
+
aborted: signal.aborted,
|
|
329
|
+
producedTokens: result.producedTokens,
|
|
330
|
+
generatedTokens: result.stats?.generatedTokens,
|
|
331
|
+
predict: mergedGenerationParams?.predict ?? modelConfig.predict
|
|
332
|
+
});
|
|
333
|
+
if (typeof kvCache === 'string') {
|
|
334
|
+
// Custom-key path: the addon wrote the new cache state inline at
|
|
335
|
+
// the same path. Either commit (records the boundary, suppresses
|
|
336
|
+
// rollback) or fall through to the deferred rollback.
|
|
337
|
+
if (shouldCommitTurn) {
|
|
338
|
+
await session.commitTurn(turn, {
|
|
339
|
+
kind: 'static',
|
|
340
|
+
messageCount: history.length + 1
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
return result;
|
|
344
|
+
}
|
|
345
|
+
// Auto-cache path.
|
|
346
|
+
//
|
|
347
|
+
// Tool-call turns: the auto-cache key is derived from
|
|
348
|
+
// `result.responseText`, which here is raw tool-call markup rather
|
|
349
|
+
// than a clean assistant message. There's no safe post-response key
|
|
350
|
+
// to rename to, so we let the deferred rollback drop the file. Once
|
|
351
|
+
// we support auto-cache for structured assistant/tool turns,
|
|
352
|
+
// this becomes a normal commit path.
|
|
353
|
+
if (result.toolCalls.length > 0) {
|
|
354
|
+
logger.warn(`[kv-cache] Auto cache tool-call turn; rolling back to avoid disk leak. path=${turn.cachePath}`);
|
|
355
|
+
return result;
|
|
356
|
+
}
|
|
357
|
+
if (!shouldCommitTurn) {
|
|
358
|
+
// Cancelled, zero-token, or budget-exhausted turns do not establish
|
|
359
|
+
// a trustworthy message boundary.
|
|
360
|
+
return result;
|
|
361
|
+
}
|
|
362
|
+
const savedHistory = buildAutoCacheSaveHistory(history.map((msg) => ({
|
|
363
|
+
role: msg.role,
|
|
364
|
+
content: msg.content,
|
|
365
|
+
attachments: msg.attachments ?? undefined
|
|
366
|
+
})), result.responseText);
|
|
367
|
+
const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory);
|
|
368
|
+
await session.commitTurn(turn, {
|
|
369
|
+
kind: 'autoRename',
|
|
370
|
+
targetCachePath: postResponseCacheInfo.cachePath,
|
|
371
|
+
messageCount: savedHistory.length
|
|
372
|
+
});
|
|
373
|
+
return result;
|
|
374
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helpers for detecting and parsing the llama.cpp addon's
|
|
3
|
+
* `ContextOverflow` status error (`LlmErrors.hpp::ContextOverflow = 14`).
|
|
4
|
+
*
|
|
5
|
+
* The addon's JS-facing throw goes through `js_throw_error(env, code,
|
|
6
|
+
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
7
|
+
* `qvac_errors::StatusError::codeString()` and formats as
|
|
8
|
+
* `"[ <addonId> :: ContextOverflow ]"`. The message carries the
|
|
9
|
+
* C++-formatted detail (which may include the prompt/ctx sizes).
|
|
10
|
+
*
|
|
11
|
+
* These helpers let the plugin handler convert that addon error into a
|
|
12
|
+
* typed `ContextOverflowError` and let unit tests assert the detection
|
|
13
|
+
* + extraction logic without a real model load.
|
|
14
|
+
*/
|
|
15
|
+
export declare function isAddonContextOverflowError(err: unknown): boolean;
|
|
16
|
+
/**
|
|
17
|
+
* Best-effort extraction of `promptTokens` / `ctxSize` from the addon
|
|
18
|
+
* error message. The C++ paths in `TextLlmContext.cpp` and
|
|
19
|
+
* `MtmdLlmContext.cpp` format both numbers into the message; the
|
|
20
|
+
* `LlamaModel::processPromptImpl` fallback path emits a bare
|
|
21
|
+
* `"<func>: context overflow\n"` with neither number. Returning
|
|
22
|
+
* `undefined` for either field is fine — `ContextOverflowError` holds
|
|
23
|
+
* them as optional and the message factory degrades gracefully.
|
|
24
|
+
*/
|
|
25
|
+
export declare function parseContextOverflowMessage(message: string): {
|
|
26
|
+
promptTokens?: number;
|
|
27
|
+
ctxSize?: number;
|
|
28
|
+
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helpers for detecting and parsing the llama.cpp addon's
|
|
3
|
+
* `ContextOverflow` status error (`LlmErrors.hpp::ContextOverflow = 14`).
|
|
4
|
+
*
|
|
5
|
+
* The addon's JS-facing throw goes through `js_throw_error(env, code,
|
|
6
|
+
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
7
|
+
* `qvac_errors::StatusError::codeString()` and formats as
|
|
8
|
+
* `"[ <addonId> :: ContextOverflow ]"`. The message carries the
|
|
9
|
+
* C++-formatted detail (which may include the prompt/ctx sizes).
|
|
10
|
+
*
|
|
11
|
+
* These helpers let the plugin handler convert that addon error into a
|
|
12
|
+
* typed `ContextOverflowError` and let unit tests assert the detection
|
|
13
|
+
* + extraction logic without a real model load.
|
|
14
|
+
*/
|
|
15
|
+
export function isAddonContextOverflowError(err) {
|
|
16
|
+
if (typeof err !== 'object' || err === null)
|
|
17
|
+
return false;
|
|
18
|
+
const code = err.code;
|
|
19
|
+
// Anchor to the codeString tail (`[ <addonId> :: ContextOverflow ]`)
|
|
20
|
+
// so we don't false-positive on a hypothetical sibling like
|
|
21
|
+
// `ContextOverflowRecovered` or `PostContextOverflow` after an
|
|
22
|
+
// addon-side rename.
|
|
23
|
+
if (typeof code === 'string' && /::\s*ContextOverflow\s*\]/.test(code)) {
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
// Message-substring fallback for the bare `LlamaModel::processPromptImpl`
|
|
27
|
+
// path, which doesn't go through `StatusError::codeString()`. Match the
|
|
28
|
+
// two known C++-emitted formats only — broader substring would catch
|
|
29
|
+
// wrapper / cause-chain text that mentions overflow without being one.
|
|
30
|
+
const message = err.message;
|
|
31
|
+
return (typeof message === 'string' &&
|
|
32
|
+
/(?:context overflow at prefill step|processPromptImpl: context overflow)/i.test(message));
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Best-effort extraction of `promptTokens` / `ctxSize` from the addon
|
|
36
|
+
* error message. The C++ paths in `TextLlmContext.cpp` and
|
|
37
|
+
* `MtmdLlmContext.cpp` format both numbers into the message; the
|
|
38
|
+
* `LlamaModel::processPromptImpl` fallback path emits a bare
|
|
39
|
+
* `"<func>: context overflow\n"` with neither number. Returning
|
|
40
|
+
* `undefined` for either field is fine — `ContextOverflowError` holds
|
|
41
|
+
* them as optional and the message factory degrades gracefully.
|
|
42
|
+
*/
|
|
43
|
+
export function parseContextOverflowMessage(message) {
|
|
44
|
+
// "[TextLlm] context overflow at prefill step: prompt tokens N, max context tokens M\n"
|
|
45
|
+
// Same-clause separator (no `[^]*?` cross-newline walk) so a future
|
|
46
|
+
// wrapper that pastes overflow text alongside unrelated numbers can't
|
|
47
|
+
// produce a mismatched pair.
|
|
48
|
+
const longForm = message.match(/prompt tokens (\d+)[,\s]+max context tokens (\d+)/i);
|
|
49
|
+
if (longForm?.[1] && longForm[2]) {
|
|
50
|
+
return {
|
|
51
|
+
promptTokens: Number(longForm[1]),
|
|
52
|
+
ctxSize: Number(longForm[2])
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
// "[TextLlm] context overflow at prefill step (N tokens, max M)\n"
|
|
56
|
+
// "[MtmdLlm] context overflow at prefill step (N tokens, max M)\n"
|
|
57
|
+
const shortForm = message.match(/\((\d+)\s+tokens,\s*max\s+(\d+)\)/i);
|
|
58
|
+
if (shortForm?.[1] && shortForm[2]) {
|
|
59
|
+
return {
|
|
60
|
+
promptTokens: Number(shortForm[1]),
|
|
61
|
+
ctxSize: Number(shortForm[2])
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
return {};
|
|
65
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { FinetuneRunParams, FinetuneRunRequest, FinetuneProgress, FinetuneRequest, FinetuneResult, FinetuneStatus, FinetuneGetStateRequest } from '../../../../schemas/index.js';
|
|
2
|
+
type FinetuneOptions = FinetuneRunParams['options'];
|
|
3
|
+
export declare function clearFinetuneRuntimeState(modelId: string): void;
|
|
4
|
+
export declare function getFinetuneStateFromCheckpoints(options: FinetuneOptions): FinetuneStatus;
|
|
5
|
+
export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
|
|
6
|
+
export declare function pauseFinetune(modelId: string): Promise<FinetuneResult>;
|
|
7
|
+
export declare function cancelFinetune(modelId: string): Promise<FinetuneResult>;
|
|
8
|
+
export declare function getFinetuneState(params: FinetuneGetStateRequest): FinetuneResult;
|
|
9
|
+
export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
|
|
10
|
+
export {};
|