@qvac/inference 0.0.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +179 -0
- package/NOTICE +986 -0
- package/README.md +138 -0
- package/dist/api/audio-gen.d.ts +18 -0
- package/dist/api/audio-gen.js +148 -0
- package/dist/api/batch-completion.d.ts +14 -0
- package/dist/api/batch-completion.js +332 -0
- package/dist/api/bci-transcribe.d.ts +56 -0
- package/dist/api/bci-transcribe.js +172 -0
- package/dist/api/cancel.d.ts +34 -0
- package/dist/api/cancel.js +82 -0
- package/dist/api/classify.d.ts +22 -0
- package/dist/api/classify.js +46 -0
- package/dist/api/completion-stream.d.ts +106 -0
- package/dist/api/completion-stream.js +351 -0
- package/dist/api/delete-cache.d.ts +29 -0
- package/dist/api/delete-cache.js +49 -0
- package/dist/api/diffusion.d.ts +92 -0
- package/dist/api/diffusion.js +176 -0
- package/dist/api/download-asset.d.ts +49 -0
- package/dist/api/download-asset.js +87 -0
- package/dist/api/embed.d.ts +41 -0
- package/dist/api/embed.js +29 -0
- package/dist/api/finetune.d.ts +89 -0
- package/dist/api/finetune.js +138 -0
- package/dist/api/get-loaded-model-info.d.ts +24 -0
- package/dist/api/get-loaded-model-info.js +34 -0
- package/dist/api/get-model-info.d.ts +43 -0
- package/dist/api/get-model-info.js +21 -0
- package/dist/api/get-system-resources.d.ts +2 -0
- package/dist/api/get-system-resources.js +16 -0
- package/dist/api/heartbeat.d.ts +29 -0
- package/dist/api/heartbeat.js +37 -0
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +35 -0
- package/dist/api/invoke-plugin.d.ts +30 -0
- package/dist/api/invoke-plugin.js +54 -0
- package/dist/api/load-model.d.ts +169 -0
- package/dist/api/load-model.js +129 -0
- package/dist/api/logging-stream-registry.d.ts +4 -0
- package/dist/api/logging-stream-registry.js +61 -0
- package/dist/api/logging-stream.d.ts +23 -0
- package/dist/api/logging-stream.js +36 -0
- package/dist/api/ocr.d.ts +30 -0
- package/dist/api/ocr.js +88 -0
- package/dist/api/parse-input.d.ts +2 -0
- package/dist/api/parse-input.js +14 -0
- package/dist/api/provide.d.ts +23 -0
- package/dist/api/provide.js +32 -0
- package/dist/api/rag.d.ts +251 -0
- package/dist/api/rag.js +485 -0
- package/dist/api/registry.d.ts +39 -0
- package/dist/api/registry.js +63 -0
- package/dist/api/resume.d.ts +29 -0
- package/dist/api/resume.js +36 -0
- package/dist/api/state.d.ts +22 -0
- package/dist/api/state.js +29 -0
- package/dist/api/stop-provider.d.ts +20 -0
- package/dist/api/stop-provider.js +30 -0
- package/dist/api/subscribe-logs.d.ts +27 -0
- package/dist/api/subscribe-logs.js +43 -0
- package/dist/api/suspend.d.ts +30 -0
- package/dist/api/suspend.js +37 -0
- package/dist/api/text-to-speech.d.ts +64 -0
- package/dist/api/text-to-speech.js +434 -0
- package/dist/api/transcribe.d.ts +75 -0
- package/dist/api/transcribe.js +256 -0
- package/dist/api/translate.d.ts +47 -0
- package/dist/api/translate.js +115 -0
- package/dist/api/unload-model.d.ts +17 -0
- package/dist/api/unload-model.js +42 -0
- package/dist/api/upscale.d.ts +44 -0
- package/dist/api/upscale.js +99 -0
- package/dist/api/video.d.ts +86 -0
- package/dist/api/video.js +176 -0
- package/dist/api/vla-helpers.d.ts +43 -0
- package/dist/api/vla-helpers.js +124 -0
- package/dist/api/vla.d.ts +68 -0
- package/dist/api/vla.js +113 -0
- package/dist/config/config-utils.d.ts +4 -0
- package/dist/config/config-utils.js +18 -0
- package/dist/config/resolve-config.d.ts +8 -0
- package/dist/config/resolve-config.js +74 -0
- package/dist/constants/audio.d.ts +1 -0
- package/dist/constants/audio.js +1 -0
- package/dist/constants/index.d.ts +2 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/registry.d.ts +1 -0
- package/dist/constants/registry.js +1 -0
- package/dist/dispatch.d.ts +18 -0
- package/dist/dispatch.js +271 -0
- package/dist/engine.d.ts +5 -0
- package/dist/engine.js +27 -0
- package/dist/errors/index.d.ts +416 -0
- package/dist/errors/index.js +681 -0
- package/dist/errors/options.d.ts +2 -0
- package/dist/errors/options.js +13 -0
- package/dist/handlers/cancelHandler.d.ts +29 -0
- package/dist/handlers/cancelHandler.js +67 -0
- package/dist/handlers/completion-orchestrate.d.ts +41 -0
- package/dist/handlers/completion-orchestrate.js +190 -0
- package/dist/handlers/delete-cache.d.ts +11 -0
- package/dist/handlers/delete-cache.js +37 -0
- package/dist/handlers/download-asset.d.ts +2 -0
- package/dist/handlers/download-asset.js +146 -0
- package/dist/handlers/finetune.d.ts +2 -0
- package/dist/handlers/finetune.js +18 -0
- package/dist/handlers/get-loaded-model-info.d.ts +2 -0
- package/dist/handlers/get-loaded-model-info.js +45 -0
- package/dist/handlers/get-model-info.d.ts +2 -0
- package/dist/handlers/get-model-info.js +173 -0
- package/dist/handlers/get-system-resources.d.ts +2 -0
- package/dist/handlers/get-system-resources.js +53 -0
- package/dist/handlers/heartbeat.d.ts +2 -0
- package/dist/handlers/heartbeat.js +3 -0
- package/dist/handlers/load-model/download-manager.d.ts +95 -0
- package/dist/handlers/load-model/download-manager.js +306 -0
- package/dist/handlers/load-model/download-stats.d.ts +7 -0
- package/dist/handlers/load-model/download-stats.js +126 -0
- package/dist/handlers/load-model/handler.d.ts +2 -0
- package/dist/handlers/load-model/handler.js +287 -0
- package/dist/handlers/load-model/http.d.ts +3 -0
- package/dist/handlers/load-model/http.js +857 -0
- package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
- package/dist/handlers/load-model/hyperdrive.js +560 -0
- package/dist/handlers/load-model/index.d.ts +2 -0
- package/dist/handlers/load-model/index.js +2 -0
- package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/handlers/load-model/registry-client-options.js +18 -0
- package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/handlers/load-model/registry-companion-set.js +206 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
- package/dist/handlers/load-model/registry-download-utils.js +112 -0
- package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
- package/dist/handlers/load-model/registry-metadata.js +23 -0
- package/dist/handlers/load-model/registry.d.ts +4 -0
- package/dist/handlers/load-model/registry.js +298 -0
- package/dist/handlers/load-model/resolve-session.d.ts +33 -0
- package/dist/handlers/load-model/resolve-session.js +62 -0
- package/dist/handlers/load-model/resolve.d.ts +5 -0
- package/dist/handlers/load-model/resolve.js +158 -0
- package/dist/handlers/load-model/types.d.ts +59 -0
- package/dist/handlers/load-model/types.js +28 -0
- package/dist/handlers/logging-stream.d.ts +2 -0
- package/dist/handlers/logging-stream.js +39 -0
- package/dist/handlers/plugin-dispatch.d.ts +3 -0
- package/dist/handlers/plugin-dispatch.js +63 -0
- package/dist/handlers/plugin-invoke.d.ts +3 -0
- package/dist/handlers/plugin-invoke.js +81 -0
- package/dist/handlers/registry.d.ts +4 -0
- package/dist/handlers/registry.js +156 -0
- package/dist/handlers/resume.d.ts +2 -0
- package/dist/handlers/resume.js +11 -0
- package/dist/handlers/state.d.ts +2 -0
- package/dist/handlers/state.js +7 -0
- package/dist/handlers/suspend.d.ts +2 -0
- package/dist/handlers/suspend.js +11 -0
- package/dist/handlers/types.d.ts +13 -0
- package/dist/handlers/types.js +1 -0
- package/dist/handlers/unload-model.d.ts +2 -0
- package/dist/handlers/unload-model.js +29 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +12 -0
- package/dist/logging/addon.d.ts +5 -0
- package/dist/logging/addon.js +75 -0
- package/dist/logging/index.d.ts +5 -0
- package/dist/logging/index.js +4 -0
- package/dist/logging/logger.d.ts +41 -0
- package/dist/logging/logger.js +209 -0
- package/dist/logging/namespaces.d.ts +6 -0
- package/dist/logging/namespaces.js +7 -0
- package/dist/logging/types.d.ts +37 -0
- package/dist/logging/types.js +1 -0
- package/dist/logging/utils.d.ts +5 -0
- package/dist/logging/utils.js +113 -0
- package/dist/models/registry/index.d.ts +18 -0
- package/dist/models/registry/index.js +21 -0
- package/dist/models/registry/models.d.ts +22680 -0
- package/dist/models/registry/models.js +23908 -0
- package/dist/p2p/cancel-delegated.d.ts +3 -0
- package/dist/p2p/cancel-delegated.js +64 -0
- package/dist/p2p/completion-stream-delegated.d.ts +4 -0
- package/dist/p2p/completion-stream-delegated.js +51 -0
- package/dist/p2p/delegate-client.d.ts +9 -0
- package/dist/p2p/delegate-client.js +219 -0
- package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
- package/dist/p2p/delegate-connect-diagnostics.js +38 -0
- package/dist/p2p/delegate-transport.d.ts +18 -0
- package/dist/p2p/delegate-transport.js +237 -0
- package/dist/p2p/error-handlers.d.ts +4 -0
- package/dist/p2p/error-handlers.js +31 -0
- package/dist/p2p/handle-request.d.ts +2 -0
- package/dist/p2p/handle-request.js +171 -0
- package/dist/p2p/handler-utils.d.ts +19 -0
- package/dist/p2p/handler-utils.js +180 -0
- package/dist/p2p/heartbeat-delegated.d.ts +3 -0
- package/dist/p2p/heartbeat-delegated.js +32 -0
- package/dist/p2p/load-model-delegated.d.ts +6 -0
- package/dist/p2p/load-model-delegated.js +132 -0
- package/dist/p2p/progress-throttle.d.ts +7 -0
- package/dist/p2p/progress-throttle.js +47 -0
- package/dist/p2p/provideHandler/connection.d.ts +2 -0
- package/dist/p2p/provideHandler/connection.js +38 -0
- package/dist/p2p/provideHandler/handler.d.ts +2 -0
- package/dist/p2p/provideHandler/handler.js +66 -0
- package/dist/p2p/provideHandler/proxy.d.ts +2 -0
- package/dist/p2p/provideHandler/proxy.js +24 -0
- package/dist/p2p/provideHandler.d.ts +1 -0
- package/dist/p2p/provideHandler.js +2 -0
- package/dist/p2p/rpc-utils.d.ts +4 -0
- package/dist/p2p/rpc-utils.js +8 -0
- package/dist/p2p/stopProvideHandler.d.ts +2 -0
- package/dist/p2p/stopProvideHandler.js +29 -0
- package/dist/p2p/swarm.d.ts +9 -0
- package/dist/p2p/swarm.js +103 -0
- package/dist/p2p/transport-selector.d.ts +2 -0
- package/dist/p2p/transport-selector.js +18 -0
- package/dist/p2p/unload-model-delegated.d.ts +2 -0
- package/dist/p2p/unload-model-delegated.js +31 -0
- package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
- package/dist/plugins/builtin/asr-ggml/config.js +46 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
- package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
- package/dist/plugins/builtin/index.d.ts +11 -0
- package/dist/plugins/builtin/index.js +11 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
- package/dist/plugins/index.d.ts +9 -0
- package/dist/plugins/index.js +12 -0
- package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
- package/dist/plugins/ops/bci-transcribe.js +248 -0
- package/dist/plugins/ops/cancel.d.ts +25 -0
- package/dist/plugins/ops/cancel.js +43 -0
- package/dist/plugins/ops/embed.d.ts +15 -0
- package/dist/plugins/ops/embed.js +193 -0
- package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
- package/dist/plugins/ops/kv-cache-retention.js +121 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
- package/dist/plugins/ops/kv-cache-utils.js +120 -0
- package/dist/plugins/ops/load-model.d.ts +8 -0
- package/dist/plugins/ops/load-model.js +82 -0
- package/dist/plugins/ops/transcribe.d.ts +20 -0
- package/dist/plugins/ops/transcribe.js +343 -0
- package/dist/plugins/ops/translate.d.ts +7 -0
- package/dist/plugins/ops/translate.js +256 -0
- package/dist/plugins/ops/unload-model.d.ts +2 -0
- package/dist/plugins/ops/unload-model.js +45 -0
- package/dist/plugins/registry.d.ts +9 -0
- package/dist/plugins/registry.js +107 -0
- package/dist/profiling/aggregator.d.ts +14 -0
- package/dist/profiling/aggregator.js +100 -0
- package/dist/profiling/backend-diagnostics.d.ts +8 -0
- package/dist/profiling/backend-diagnostics.js +36 -0
- package/dist/profiling/clock.d.ts +8 -0
- package/dist/profiling/clock.js +23 -0
- package/dist/profiling/controller.d.ts +34 -0
- package/dist/profiling/controller.js +130 -0
- package/dist/profiling/delegation-profiler.d.ts +31 -0
- package/dist/profiling/delegation-profiler.js +155 -0
- package/dist/profiling/envelope.d.ts +10 -0
- package/dist/profiling/envelope.js +45 -0
- package/dist/profiling/events.d.ts +19 -0
- package/dist/profiling/events.js +35 -0
- package/dist/profiling/exporters.d.ts +9 -0
- package/dist/profiling/exporters.js +242 -0
- package/dist/profiling/index.d.ts +91 -0
- package/dist/profiling/index.js +87 -0
- package/dist/profiling/model-execution.d.ts +15 -0
- package/dist/profiling/model-execution.js +34 -0
- package/dist/profiling/operation-metrics.d.ts +24 -0
- package/dist/profiling/operation-metrics.js +370 -0
- package/dist/profiling/operation-wrappers.d.ts +17 -0
- package/dist/profiling/operation-wrappers.js +196 -0
- package/dist/profiling/profiler.d.ts +27 -0
- package/dist/profiling/profiler.js +127 -0
- package/dist/profiling/ring-buffer.d.ts +18 -0
- package/dist/profiling/ring-buffer.js +83 -0
- package/dist/profiling/types.d.ts +52 -0
- package/dist/profiling/types.js +1 -0
- package/dist/rag/handler.d.ts +2 -0
- package/dist/rag/handler.js +272 -0
- package/dist/rag/handlers/chunk.d.ts +2 -0
- package/dist/rag/handlers/chunk.js +7 -0
- package/dist/rag/handlers/close-workspace.d.ts +2 -0
- package/dist/rag/handlers/close-workspace.js +9 -0
- package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
- package/dist/rag/handlers/delete-embeddings.js +11 -0
- package/dist/rag/handlers/delete-workspace.d.ts +2 -0
- package/dist/rag/handlers/delete-workspace.js +14 -0
- package/dist/rag/handlers/ingest.d.ts +9 -0
- package/dist/rag/handlers/ingest.js +19 -0
- package/dist/rag/handlers/list-workspaces.d.ts +2 -0
- package/dist/rag/handlers/list-workspaces.js +4 -0
- package/dist/rag/handlers/reindex.d.ts +9 -0
- package/dist/rag/handlers/reindex.js +16 -0
- package/dist/rag/handlers/save-embeddings.d.ts +9 -0
- package/dist/rag/handlers/save-embeddings.js +15 -0
- package/dist/rag/handlers/search.d.ts +2 -0
- package/dist/rag/handlers/search.js +16 -0
- package/dist/rag/index.d.ts +11 -0
- package/dist/rag/index.js +11 -0
- package/dist/rag/rag-operation-manager.d.ts +28 -0
- package/dist/rag/rag-operation-manager.js +76 -0
- package/dist/rag/rag-workspace-manager.d.ts +14 -0
- package/dist/rag/rag-workspace-manager.js +143 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +166 -0
- package/dist/resources/collector.d.ts +643 -0
- package/dist/resources/collector.js +226 -0
- package/dist/resources/instance.d.ts +4 -0
- package/dist/resources/instance.js +15 -0
- package/dist/resources/native.d.ts +2 -0
- package/dist/resources/native.js +35 -0
- package/dist/resources/normalize.d.ts +12 -0
- package/dist/resources/normalize.js +73 -0
- package/dist/resources/types.d.ts +65 -0
- package/dist/resources/types.js +1 -0
- package/dist/runtime/cache-lock.d.ts +2 -0
- package/dist/runtime/cache-lock.js +67 -0
- package/dist/runtime/disposable-scope.d.ts +35 -0
- package/dist/runtime/disposable-scope.js +69 -0
- package/dist/runtime/env.d.ts +16 -0
- package/dist/runtime/env.js +53 -0
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +20 -0
- package/dist/runtime/lifecycle.js +102 -0
- package/dist/runtime/logging-stream-registry.d.ts +27 -0
- package/dist/runtime/logging-stream-registry.js +133 -0
- package/dist/runtime/model-config-utils.d.ts +14 -0
- package/dist/runtime/model-config-utils.js +128 -0
- package/dist/runtime/model-registry.d.ts +74 -0
- package/dist/runtime/model-registry.js +130 -0
- package/dist/runtime/registry-client.d.ts +4 -0
- package/dist/runtime/registry-client.js +118 -0
- package/dist/runtime/request-context.d.ts +72 -0
- package/dist/runtime/request-context.js +114 -0
- package/dist/runtime/request-id.d.ts +17 -0
- package/dist/runtime/request-id.js +29 -0
- package/dist/runtime/request-registry.d.ts +194 -0
- package/dist/runtime/request-registry.js +546 -0
- package/dist/runtime/runtime-lifecycle.d.ts +37 -0
- package/dist/runtime/runtime-lifecycle.js +163 -0
- package/dist/runtime/state.d.ts +26 -0
- package/dist/runtime/state.js +138 -0
- package/dist/schemas/audio-gen.d.ts +252 -0
- package/dist/schemas/audio-gen.js +69 -0
- package/dist/schemas/batch-completion-stream.d.ts +467 -0
- package/dist/schemas/batch-completion-stream.js +123 -0
- package/dist/schemas/bci-config.d.ts +74 -0
- package/dist/schemas/bci-config.js +58 -0
- package/dist/schemas/bci.d.ts +178 -0
- package/dist/schemas/bci.js +89 -0
- package/dist/schemas/cancel.d.ts +203 -0
- package/dist/schemas/cancel.js +135 -0
- package/dist/schemas/classification.d.ts +52 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +6420 -0
- package/dist/schemas/common.js +133 -0
- package/dist/schemas/completion-event.d.ts +248 -0
- package/dist/schemas/completion-event.js +94 -0
- package/dist/schemas/completion-stream.d.ts +475 -0
- package/dist/schemas/completion-stream.js +236 -0
- package/dist/schemas/config.d.ts +5022 -0
- package/dist/schemas/config.js +169 -0
- package/dist/schemas/delegate.d.ts +29 -0
- package/dist/schemas/delegate.js +40 -0
- package/dist/schemas/download-asset.d.ts +225 -0
- package/dist/schemas/download-asset.js +71 -0
- package/dist/schemas/embed.d.ts +41 -0
- package/dist/schemas/embed.js +35 -0
- package/dist/schemas/engine-addon-map.d.ts +20 -0
- package/dist/schemas/engine-addon-map.js +78 -0
- package/dist/schemas/error.d.ts +40 -0
- package/dist/schemas/error.js +65 -0
- package/dist/schemas/errors.d.ts +119 -0
- package/dist/schemas/errors.js +615 -0
- package/dist/schemas/finetune.d.ts +621 -0
- package/dist/schemas/finetune.js +163 -0
- package/dist/schemas/index.d.ts +45 -0
- package/dist/schemas/index.js +45 -0
- package/dist/schemas/json-schema.d.ts +15 -0
- package/dist/schemas/json-schema.js +2 -0
- package/dist/schemas/lifecycle.d.ts +38 -0
- package/dist/schemas/lifecycle.js +21 -0
- package/dist/schemas/llamacpp-config.d.ts +403 -0
- package/dist/schemas/llamacpp-config.js +133 -0
- package/dist/schemas/load-model.d.ts +25635 -0
- package/dist/schemas/load-model.js +629 -0
- package/dist/schemas/logging-stream.d.ts +33 -0
- package/dist/schemas/logging-stream.js +17 -0
- package/dist/schemas/mcp-adapter.d.ts +46 -0
- package/dist/schemas/mcp-adapter.js +1 -0
- package/dist/schemas/model-file.d.ts +27 -0
- package/dist/schemas/model-file.js +29 -0
- package/dist/schemas/model-info.d.ts +197 -0
- package/dist/schemas/model-info.js +144 -0
- package/dist/schemas/model-ops.d.ts +36 -0
- package/dist/schemas/model-ops.js +40 -0
- package/dist/schemas/model-src-utils.d.ts +192 -0
- package/dist/schemas/model-src-utils.js +61 -0
- package/dist/schemas/model-types.d.ts +275 -0
- package/dist/schemas/model-types.js +223 -0
- package/dist/schemas/ocr.d.ts +138 -0
- package/dist/schemas/ocr.js +59 -0
- package/dist/schemas/path-validation.d.ts +9 -0
- package/dist/schemas/path-validation.js +22 -0
- package/dist/schemas/plugin.d.ts +341 -0
- package/dist/schemas/plugin.js +241 -0
- package/dist/schemas/profiling.d.ts +695 -0
- package/dist/schemas/profiling.js +100 -0
- package/dist/schemas/provide.d.ts +50 -0
- package/dist/schemas/provide.js +32 -0
- package/dist/schemas/rag.d.ts +301 -0
- package/dist/schemas/rag.js +211 -0
- package/dist/schemas/registry.d.ts +453 -0
- package/dist/schemas/registry.js +104 -0
- package/dist/schemas/reload-config.d.ts +585 -0
- package/dist/schemas/reload-config.js +42 -0
- package/dist/schemas/runtime-context.d.ts +18 -0
- package/dist/schemas/runtime-context.js +7 -0
- package/dist/schemas/sdcpp-config.d.ts +1126 -0
- package/dist/schemas/sdcpp-config.js +748 -0
- package/dist/schemas/system-resources.d.ts +3966 -0
- package/dist/schemas/system-resources.js +149 -0
- package/dist/schemas/text-to-speech.d.ts +1352 -0
- package/dist/schemas/text-to-speech.js +364 -0
- package/dist/schemas/tools.d.ts +80 -0
- package/dist/schemas/tools.js +52 -0
- package/dist/schemas/transcription-config.d.ts +161 -0
- package/dist/schemas/transcription-config.js +156 -0
- package/dist/schemas/transcription.d.ts +261 -0
- package/dist/schemas/transcription.js +145 -0
- package/dist/schemas/translate.d.ts +138 -0
- package/dist/schemas/translate.js +103 -0
- package/dist/schemas/translation-config.d.ts +1031 -0
- package/dist/schemas/translation-config.js +170 -0
- package/dist/schemas/vla.d.ts +110 -0
- package/dist/schemas/vla.js +151 -0
- package/dist/selection.d.ts +8 -0
- package/dist/selection.js +14 -0
- package/dist/surface.d.ts +18 -0
- package/dist/surface.js +30 -0
- package/dist/utils/addon-responses.d.ts +58 -0
- package/dist/utils/addon-responses.js +1 -0
- package/dist/utils/aggregate-events.d.ts +25 -0
- package/dist/utils/aggregate-events.js +73 -0
- package/dist/utils/archive.d.ts +4 -0
- package/dist/utils/archive.js +180 -0
- package/dist/utils/asr-events.d.ts +14 -0
- package/dist/utils/asr-events.js +30 -0
- package/dist/utils/audio/decoder.d.ts +4 -0
- package/dist/utils/audio/decoder.js +80 -0
- package/dist/utils/audio-input.d.ts +9 -0
- package/dist/utils/audio-input.js +34 -0
- package/dist/utils/cache/auto-history.d.ts +3 -0
- package/dist/utils/cache/auto-history.js +26 -0
- package/dist/utils/cache/paths.d.ts +42 -0
- package/dist/utils/cache/paths.js +133 -0
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/cache/types.js +1 -0
- package/dist/utils/cache-normalize.d.ts +9 -0
- package/dist/utils/cache-normalize.js +20 -0
- package/dist/utils/checksum.d.ts +3 -0
- package/dist/utils/checksum.js +29 -0
- package/dist/utils/clear-storage.d.ts +4 -0
- package/dist/utils/clear-storage.js +14 -0
- package/dist/utils/completion-normalizer.d.ts +17 -0
- package/dist/utils/completion-normalizer.js +453 -0
- package/dist/utils/config-transform.d.ts +2 -0
- package/dist/utils/config-transform.js +21 -0
- package/dist/utils/decorate-promise.d.ts +33 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/encoding.d.ts +16 -0
- package/dist/utils/encoding.js +27 -0
- package/dist/utils/expand-gguf-shards.d.ts +2 -0
- package/dist/utils/expand-gguf-shards.js +22 -0
- package/dist/utils/formatting.d.ts +18 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
- package/dist/utils/gguf-tensor-extractor.js +311 -0
- package/dist/utils/global-singleton.d.ts +1 -0
- package/dist/utils/global-singleton.js +10 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/load-model-validation.d.ts +6 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/dist/utils/mcp-adapter.d.ts +11 -0
- package/dist/utils/mcp-adapter.js +96 -0
- package/dist/utils/multi-gpu-mobile.d.ts +4 -0
- package/dist/utils/multi-gpu-mobile.js +10 -0
- package/dist/utils/object.d.ts +1 -0
- package/dist/utils/object.js +3 -0
- package/dist/utils/path-security.d.ts +22 -0
- package/dist/utils/path-security.js +69 -0
- package/dist/utils/qvac-paths.d.ts +1 -0
- package/dist/utils/qvac-paths.js +5 -0
- package/dist/utils/response-format.d.ts +2 -0
- package/dist/utils/response-format.js +20 -0
- package/dist/utils/shard-utils.d.ts +47 -0
- package/dist/utils/shard-utils.js +123 -0
- package/dist/utils/tool-helpers.d.ts +24 -0
- package/dist/utils/tool-helpers.js +111 -0
- package/dist/utils/tool-integration.d.ts +23 -0
- package/dist/utils/tool-integration.js +29 -0
- package/dist/utils/tools/dialect.d.ts +7 -0
- package/dist/utils/tools/dialect.js +20 -0
- package/dist/utils/tools/index.d.ts +2 -0
- package/dist/utils/tools/index.js +2 -0
- package/dist/utils/tools/parser.d.ts +5 -0
- package/dist/utils/tools/parser.js +69 -0
- package/dist/utils/tools/parsers/dsml.d.ts +3 -0
- package/dist/utils/tools/parsers/dsml.js +177 -0
- package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
- package/dist/utils/tools/parsers/gemma4native.js +65 -0
- package/dist/utils/tools/parsers/harmony.d.ts +3 -0
- package/dist/utils/tools/parsers/harmony.js +57 -0
- package/dist/utils/tools/parsers/hermes.d.ts +3 -0
- package/dist/utils/tools/parsers/hermes.js +122 -0
- package/dist/utils/tools/parsers/json.d.ts +5 -0
- package/dist/utils/tools/parsers/json.js +115 -0
- package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
- package/dist/utils/tools/parsers/pythonic.js +267 -0
- package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
- package/dist/utils/tools/parsers/qwen35.js +125 -0
- package/dist/utils/tools/shared.d.ts +17 -0
- package/dist/utils/tools/shared.js +52 -0
- package/dist/utils/transcribe-metadata.d.ts +20 -0
- package/dist/utils/transcribe-metadata.js +28 -0
- package/dist/utils/tts-stats.d.ts +25 -0
- package/dist/utils/tts-stats.js +16 -0
- package/dist/utils/withTimeout.d.ts +17 -0
- package/dist/utils/withTimeout.js +47 -0
- package/dist/utils/zod-error.d.ts +2 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +266 -1
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type OCRConfig, type ResolveContext, type ResolveResult } from '../../../schemas/index.js';
|
|
2
|
+
type OcrPipelineType = NonNullable<OCRConfig['pipelineType']>;
|
|
3
|
+
export declare const DETECTOR_FILENAMES: Record<OcrPipelineType, string>;
|
|
4
|
+
export declare function resolveOcrConfig(cfg: OCRConfig, ctx: ResolveContext): Promise<ResolveResult<Record<string, unknown>, 'detectorModelPath'>>;
|
|
5
|
+
export {};
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { ModelLoadFailedError } from '../../../errors/index.js';
|
|
2
|
+
import { hyperdriveUrlSchema } from '../../../schemas/load-model.js';
|
|
3
|
+
import { OCR_CRAFT, OCR_DOCTR_1 } from '../../../models/registry/index.js';
|
|
4
|
+
// Detector weights paired with each pipeline. EasyOCR runs CRAFT
|
|
5
|
+
// detection, DocTR runs DBNet detection — the addon cannot load one
|
|
6
|
+
// pipeline's detector into the other.
|
|
7
|
+
export const DETECTOR_FILENAMES = {
|
|
8
|
+
easyocr: 'craft_mlt_25k.gguf',
|
|
9
|
+
doctr: 'db_mobilenet_v3_large.gguf'
|
|
10
|
+
};
|
|
11
|
+
const DETECTOR_REGISTRY_SOURCES = {
|
|
12
|
+
easyocr: OCR_CRAFT.src,
|
|
13
|
+
doctr: OCR_DOCTR_1.src
|
|
14
|
+
};
|
|
15
|
+
// The addon defaults to the EasyOCR pipeline, so a DocTR recognizer
|
|
16
|
+
// loaded without an explicit `pipelineType` dies deep in the EasyOCR
|
|
17
|
+
// CRNN weight loader with a raw missing-tensor error (QVAC-22514).
|
|
18
|
+
// Recognize the known DocTR registry artifacts by name and opt in to
|
|
19
|
+
// the DocTR pipeline automatically.
|
|
20
|
+
function inferPipelineType(recognizerSrc) {
|
|
21
|
+
const path = srcPathname(recognizerSrc);
|
|
22
|
+
const filename = path.split('/').pop() ?? '';
|
|
23
|
+
if (filename === 'crnn_mobilenet_v3_small.gguf' || path.includes('/gguf/doctr')) {
|
|
24
|
+
return 'doctr';
|
|
25
|
+
}
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
function srcPathname(src) {
|
|
29
|
+
return (src.split(/[?#]/)[0] ?? '').toLowerCase();
|
|
30
|
+
}
|
|
31
|
+
function srcToString(src) {
|
|
32
|
+
return typeof src === 'string' ? src : src.src;
|
|
33
|
+
}
|
|
34
|
+
// The registry still lists legacy `.onnx` OCR artifacts under the
|
|
35
|
+
// "@qvac/ocr-onnx" engine; the ggml-ocr addon can only open GGUF, so a
|
|
36
|
+
// raw "failed to open GGUF" surfaces mid-load. Fail fast with the
|
|
37
|
+
// supported configurations instead (QVAC-22514).
|
|
38
|
+
function assertGgufSource(src, role) {
|
|
39
|
+
if (!srcPathname(src).endsWith('.onnx'))
|
|
40
|
+
return;
|
|
41
|
+
throw new ModelLoadFailedError(`ggml-ocr only loads GGUF models, but the ${role} source points to an ONNX file: ${src}. ` +
|
|
42
|
+
'The .onnx OCR registry entries are legacy artifacts of the removed "onnx-ocr" engine. ' +
|
|
43
|
+
'Supported registry configurations: EasyOCR (default) — modelSrc: OCR_LATIN.src (detector OCR_CRAFT auto-derived); ' +
|
|
44
|
+
'DocTR — modelSrc: OCR_DOCTR.src (pipelineType and detector OCR_DOCTR_1 auto-derived).');
|
|
45
|
+
}
|
|
46
|
+
function deriveDetectorSource(modelSrc, pipelineType) {
|
|
47
|
+
if (modelSrc.startsWith('pear://')) {
|
|
48
|
+
const { key } = hyperdriveUrlSchema.parse(modelSrc);
|
|
49
|
+
return `pear://${key}/${DETECTOR_FILENAMES[pipelineType]}`;
|
|
50
|
+
}
|
|
51
|
+
if (modelSrc.startsWith('registry://')) {
|
|
52
|
+
return DETECTOR_REGISTRY_SOURCES[pipelineType];
|
|
53
|
+
}
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
export async function resolveOcrConfig(cfg, ctx) {
|
|
57
|
+
const { detectorModelSrc, ...ocrConfig } = cfg;
|
|
58
|
+
assertGgufSource(ctx.modelSrc, 'recognizer');
|
|
59
|
+
const pipelineType = ocrConfig.pipelineType ?? inferPipelineType(ctx.modelSrc) ?? 'easyocr';
|
|
60
|
+
const detectorSrc = detectorModelSrc ?? deriveDetectorSource(ctx.modelSrc, pipelineType);
|
|
61
|
+
if (!detectorSrc) {
|
|
62
|
+
throw new ModelLoadFailedError(`Detector model required for OCR (${pipelineType} pipeline expects ` +
|
|
63
|
+
`${DETECTOR_FILENAMES[pipelineType]}). Use a hyperdrive or registry ` +
|
|
64
|
+
'source, or provide detectorModelSrc');
|
|
65
|
+
}
|
|
66
|
+
assertGgufSource(srcToString(detectorSrc), 'detector');
|
|
67
|
+
const detectorModelPath = await ctx.resolveModelPath(detectorSrc);
|
|
68
|
+
return {
|
|
69
|
+
config: { ...ocrConfig, pipelineType },
|
|
70
|
+
artifacts: { detectorModelPath }
|
|
71
|
+
};
|
|
72
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { getModel } from '../../../../runtime/model-registry.js';
|
|
2
|
+
import { vlaHparamsSchema } from '../../../../schemas/index.js';
|
|
3
|
+
export function vlaGetHparams(request) {
|
|
4
|
+
const model = getModel(request.modelId);
|
|
5
|
+
// Validate the addon-reported hparams against our schema so the response
|
|
6
|
+
// shape stays consistent even if the underlying addon changes.
|
|
7
|
+
const parsed = vlaHparamsSchema.parse(model.hparams);
|
|
8
|
+
return Promise.resolve({
|
|
9
|
+
hparams: parsed,
|
|
10
|
+
backendName: model.backendName ?? null
|
|
11
|
+
});
|
|
12
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { getModel } from '../../../../runtime/model-registry.js';
|
|
2
|
+
import { decodeBase64, encodeBase64 } from '../../../../utils/encoding.js';
|
|
3
|
+
function f32FromBase64(b64) {
|
|
4
|
+
const bytes = decodeBase64(b64);
|
|
5
|
+
// ArrayBuffer view must be aligned; copy if needed.
|
|
6
|
+
if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
|
|
7
|
+
return new Float32Array(bytes.buffer);
|
|
8
|
+
}
|
|
9
|
+
const copy = new Uint8Array(bytes.byteLength);
|
|
10
|
+
copy.set(bytes);
|
|
11
|
+
return new Float32Array(copy.buffer);
|
|
12
|
+
}
|
|
13
|
+
function i32FromBase64(b64) {
|
|
14
|
+
const bytes = decodeBase64(b64);
|
|
15
|
+
if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
|
|
16
|
+
return new Int32Array(bytes.buffer);
|
|
17
|
+
}
|
|
18
|
+
const copy = new Uint8Array(bytes.byteLength);
|
|
19
|
+
copy.set(bytes);
|
|
20
|
+
return new Int32Array(copy.buffer);
|
|
21
|
+
}
|
|
22
|
+
function u8FromBase64(b64) {
|
|
23
|
+
return decodeBase64(b64);
|
|
24
|
+
}
|
|
25
|
+
function f32ToBase64(arr) {
|
|
26
|
+
return encodeBase64(new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength));
|
|
27
|
+
}
|
|
28
|
+
function pickStats(raw) {
|
|
29
|
+
if (!raw || typeof raw !== 'object')
|
|
30
|
+
return undefined;
|
|
31
|
+
const r = raw;
|
|
32
|
+
const out = {};
|
|
33
|
+
if (typeof r['vision_ms'] === 'number')
|
|
34
|
+
out.vision_ms = r['vision_ms'];
|
|
35
|
+
if (typeof r['prefill_compute_ms'] === 'number')
|
|
36
|
+
out.prefill_compute_ms = r['prefill_compute_ms'];
|
|
37
|
+
if (typeof r['prefill_total_ms'] === 'number')
|
|
38
|
+
out.prefill_total_ms = r['prefill_total_ms'];
|
|
39
|
+
if (typeof r['smollm2_compute_ms'] === 'number')
|
|
40
|
+
out.smollm2_compute_ms = r['smollm2_compute_ms'];
|
|
41
|
+
if (typeof r['smollm2_total_ms'] === 'number')
|
|
42
|
+
out.smollm2_total_ms = r['smollm2_total_ms'];
|
|
43
|
+
if (typeof r['ode_ms'] === 'number')
|
|
44
|
+
out.ode_ms = r['ode_ms'];
|
|
45
|
+
if (typeof r['total_ms'] === 'number')
|
|
46
|
+
out.total_ms = r['total_ms'];
|
|
47
|
+
if (typeof r['backendDevice'] === 'number')
|
|
48
|
+
out.backendDevice = r['backendDevice'];
|
|
49
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
50
|
+
}
|
|
51
|
+
export async function vlaRun(request) {
|
|
52
|
+
const model = getModel(request.modelId);
|
|
53
|
+
const images = request.images.map(f32FromBase64);
|
|
54
|
+
const state = f32FromBase64(request.state);
|
|
55
|
+
const tokens = i32FromBase64(request.tokens);
|
|
56
|
+
const mask = u8FromBase64(request.mask);
|
|
57
|
+
const noise = request.noise ? f32FromBase64(request.noise) : undefined;
|
|
58
|
+
const response = await model.run({
|
|
59
|
+
images,
|
|
60
|
+
imgWidth: request.imgWidth,
|
|
61
|
+
imgHeight: request.imgHeight,
|
|
62
|
+
state,
|
|
63
|
+
tokens,
|
|
64
|
+
mask,
|
|
65
|
+
...(noise && { noise })
|
|
66
|
+
});
|
|
67
|
+
const result = await response.await();
|
|
68
|
+
const hp = model.hparams;
|
|
69
|
+
if (!hp) {
|
|
70
|
+
throw new Error('VLA model hparams unavailable after run');
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
actions: f32ToBase64(result.actions),
|
|
74
|
+
actionDim: hp.actionDim,
|
|
75
|
+
chunkSize: hp.chunkSize,
|
|
76
|
+
...(pickStats(result.stats) && { stats: pickStats(result.stats) })
|
|
77
|
+
};
|
|
78
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
|
|
2
|
+
export declare const vlaPlugin: {
|
|
3
|
+
modelType: "ggml-vla";
|
|
4
|
+
displayName: string;
|
|
5
|
+
addonPackage: "@qvac/vla-ggml";
|
|
6
|
+
loadConfigSchema: import("zod").ZodObject<{
|
|
7
|
+
backend: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
8
|
+
cpu: "cpu";
|
|
9
|
+
auto: "auto";
|
|
10
|
+
}>>;
|
|
11
|
+
verbosity: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
12
|
+
}, import("zod/v4/core").$strip>;
|
|
13
|
+
createModel(params: CreateModelParams): PluginModelResult;
|
|
14
|
+
handlers: {
|
|
15
|
+
vlaRun: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
16
|
+
type: import("zod").ZodLiteral<"vlaRun">;
|
|
17
|
+
modelId: import("zod").ZodString;
|
|
18
|
+
images: import("zod").ZodArray<import("zod").ZodString>;
|
|
19
|
+
imgWidth: import("zod").ZodNumber;
|
|
20
|
+
imgHeight: import("zod").ZodNumber;
|
|
21
|
+
state: import("zod").ZodString;
|
|
22
|
+
tokens: import("zod").ZodString;
|
|
23
|
+
mask: import("zod").ZodString;
|
|
24
|
+
noise: import("zod").ZodOptional<import("zod").ZodString>;
|
|
25
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
26
|
+
actions: import("zod").ZodString;
|
|
27
|
+
actionDim: import("zod").ZodNumber;
|
|
28
|
+
chunkSize: import("zod").ZodNumber;
|
|
29
|
+
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
30
|
+
vision_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
31
|
+
prefill_compute_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
32
|
+
prefill_total_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
33
|
+
smollm2_compute_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
34
|
+
smollm2_total_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
35
|
+
ode_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
36
|
+
total_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
37
|
+
backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
38
|
+
}, import("zod/v4/core").$strip>>;
|
|
39
|
+
}, import("zod/v4/core").$strip>>;
|
|
40
|
+
vlaHparams: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
41
|
+
type: import("zod").ZodLiteral<"vlaHparams">;
|
|
42
|
+
modelId: import("zod").ZodString;
|
|
43
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
44
|
+
hparams: import("zod").ZodObject<{
|
|
45
|
+
chunkSize: import("zod").ZodNumber;
|
|
46
|
+
actionDim: import("zod").ZodNumber;
|
|
47
|
+
maxActionDim: import("zod").ZodNumber;
|
|
48
|
+
maxStateDim: import("zod").ZodNumber;
|
|
49
|
+
tokenizerMaxLength: import("zod").ZodNumber;
|
|
50
|
+
visionImageSize: import("zod").ZodNumber;
|
|
51
|
+
numCameras: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
52
|
+
stateInputMode: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
53
|
+
discrete: "discrete";
|
|
54
|
+
continuous: "continuous";
|
|
55
|
+
}>>;
|
|
56
|
+
imageInputMode: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
57
|
+
pixels: "pixels";
|
|
58
|
+
patches: "patches";
|
|
59
|
+
}>>;
|
|
60
|
+
imagePatchElems: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
61
|
+
}, import("zod/v4/core").$strip>;
|
|
62
|
+
backendName: import("zod").ZodNullable<import("zod").ZodString>;
|
|
63
|
+
}, import("zod/v4/core").$strip>>;
|
|
64
|
+
};
|
|
65
|
+
};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { VlaModel } from '@qvac/vla-ggml';
|
|
2
|
+
import { definePlugin, defineHandler, vlaConfigSchema, vlaRunRequestSchema, vlaRunResponseSchema, vlaHparamsRequestSchema, vlaHparamsResponseSchema, ModelType, ADDON_VLA } from '../../../schemas/index.js';
|
|
3
|
+
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
4
|
+
import { vlaRun } from '../../../plugins/builtin/ggml-vla/ops/vla-run.js';
|
|
5
|
+
import { vlaGetHparams } from '../../../plugins/builtin/ggml-vla/ops/vla-hparams.js';
|
|
6
|
+
// The `@qvac/vla-ggml` VlaModel exposes `load({ backend })` rather than the
|
|
7
|
+
// `load(force?)` signature `PluginModel` expects. Wrap it so the plugin
|
|
8
|
+
// framework can call `load()` and have the configured backend flow through.
|
|
9
|
+
function wrapVlaModel(inner, loadOpts) {
|
|
10
|
+
const wrapper = inner;
|
|
11
|
+
const originalLoad = wrapper.load.bind(wrapper);
|
|
12
|
+
wrapper.load = function load() {
|
|
13
|
+
return originalLoad(loadOpts);
|
|
14
|
+
};
|
|
15
|
+
return wrapper;
|
|
16
|
+
}
|
|
17
|
+
export const vlaPlugin = definePlugin({
|
|
18
|
+
modelType: ModelType.ggmlVla,
|
|
19
|
+
displayName: 'VLA (SmolVLA / π₀.₅ / GR00T ggml)',
|
|
20
|
+
addonPackage: ADDON_VLA,
|
|
21
|
+
loadConfigSchema: vlaConfigSchema,
|
|
22
|
+
createModel(params) {
|
|
23
|
+
const config = (params.modelConfig ?? {});
|
|
24
|
+
const logger = createStreamLogger(params.modelId, ModelType.ggmlVla);
|
|
25
|
+
registerAddonLogger(params.modelId, ModelType.ggmlVla, logger);
|
|
26
|
+
const inner = new VlaModel({
|
|
27
|
+
files: { model: [params.modelPath] },
|
|
28
|
+
...(config.verbosity !== undefined && {
|
|
29
|
+
config: { verbosity: config.verbosity }
|
|
30
|
+
}),
|
|
31
|
+
logger,
|
|
32
|
+
opts: { stats: true }
|
|
33
|
+
});
|
|
34
|
+
const backend = config.backend ?? 'auto';
|
|
35
|
+
const model = wrapVlaModel(inner, { backend });
|
|
36
|
+
return { model };
|
|
37
|
+
},
|
|
38
|
+
handlers: {
|
|
39
|
+
vlaRun: defineHandler({
|
|
40
|
+
requestSchema: vlaRunRequestSchema,
|
|
41
|
+
responseSchema: vlaRunResponseSchema,
|
|
42
|
+
streaming: false,
|
|
43
|
+
// The vla-ggml addon exposes a model-wide cancel(): the running ODE /
|
|
44
|
+
// SmolLM2 prefill is interrupted. Mirrors the diffusion plugin's
|
|
45
|
+
// cancel surface.
|
|
46
|
+
cancel: { scope: 'model', hard: true },
|
|
47
|
+
handler: vlaRun
|
|
48
|
+
}),
|
|
49
|
+
vlaHparams: defineHandler({
|
|
50
|
+
requestSchema: vlaHparamsRequestSchema,
|
|
51
|
+
responseSchema: vlaHparamsResponseSchema,
|
|
52
|
+
streaming: false,
|
|
53
|
+
cancel: { scope: 'none' },
|
|
54
|
+
handler: vlaGetHparams
|
|
55
|
+
})
|
|
56
|
+
}
|
|
57
|
+
});
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { llmPlugin } from '../../plugins/builtin/llamacpp-completion/plugin.js';
|
|
2
|
+
export { embeddingsPlugin } from '../../plugins/builtin/llamacpp-embedding/plugin.js';
|
|
3
|
+
export { whisperPlugin } from '../../plugins/builtin/whispercpp-transcription/plugin.js';
|
|
4
|
+
export { bciPlugin } from '../../plugins/builtin/bci-whispercpp-transcription/plugin.js';
|
|
5
|
+
export { parakeetPlugin } from '../../plugins/builtin/parakeet-transcription/plugin.js';
|
|
6
|
+
export { nmtPlugin } from '../../plugins/builtin/nmtcpp-translation/plugin.js';
|
|
7
|
+
export { ttsPlugin } from '../../plugins/builtin/tts-ggml/plugin.js';
|
|
8
|
+
export { ocrPlugin } from '../../plugins/builtin/ggml-ocr/plugin.js';
|
|
9
|
+
export { diffusionPlugin } from '../../plugins/builtin/sdcpp-generation/plugin.js';
|
|
10
|
+
export { vlaPlugin } from '../../plugins/builtin/ggml-vla/plugin.js';
|
|
11
|
+
export { classificationPlugin } from '../../plugins/builtin/ggml-classification/plugin.js';
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { llmPlugin } from '../../plugins/builtin/llamacpp-completion/plugin.js';
|
|
2
|
+
export { embeddingsPlugin } from '../../plugins/builtin/llamacpp-embedding/plugin.js';
|
|
3
|
+
export { whisperPlugin } from '../../plugins/builtin/whispercpp-transcription/plugin.js';
|
|
4
|
+
export { bciPlugin } from '../../plugins/builtin/bci-whispercpp-transcription/plugin.js';
|
|
5
|
+
export { parakeetPlugin } from '../../plugins/builtin/parakeet-transcription/plugin.js';
|
|
6
|
+
export { nmtPlugin } from '../../plugins/builtin/nmtcpp-translation/plugin.js';
|
|
7
|
+
export { ttsPlugin } from '../../plugins/builtin/tts-ggml/plugin.js';
|
|
8
|
+
export { ocrPlugin } from '../../plugins/builtin/ggml-ocr/plugin.js';
|
|
9
|
+
export { diffusionPlugin } from '../../plugins/builtin/sdcpp-generation/plugin.js';
|
|
10
|
+
export { vlaPlugin } from '../../plugins/builtin/ggml-vla/plugin.js';
|
|
11
|
+
export { classificationPlugin } from '../../plugins/builtin/ggml-classification/plugin.js';
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helper for detecting the llama.cpp addon's `Cancelled` status error
|
|
3
|
+
* (`LlmErrors.hpp::Cancelled = 27`) on the continuous-batching path.
|
|
4
|
+
*
|
|
5
|
+
* When a batch is cancelled while it still has prompts queued behind the
|
|
6
|
+
* `parallel` slot limit, `ContinuousBatchScheduler::cancelPendingLocked`
|
|
7
|
+
* fails the whole batch group with a `Cancelled` `StatusError`, which
|
|
8
|
+
* `processBatch` rethrows — so `model.run(prompts)` rejects rather than
|
|
9
|
+
* resolving with partial outputs (see
|
|
10
|
+
* `packages/llm-llamacpp/docs/continuous-batching.md` "Cancellation
|
|
11
|
+
* semantics"). In-flight slots are still cancelled gracefully, but the
|
|
12
|
+
* shared group carries the error.
|
|
13
|
+
*
|
|
14
|
+
* The addon's JS-facing throw goes through `js_throw_error(env, code,
|
|
15
|
+
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
16
|
+
* `qvac_errors::StatusError::codeString()` and formats as
|
|
17
|
+
* `"[ <addonId> :: Cancelled ]"`. This helper lets the batch handler
|
|
18
|
+
* recognise that rejection so it can complete the run with cancelled
|
|
19
|
+
* terminal events (matching the graceful cancel semantics) instead
|
|
20
|
+
* of surfacing a generic `CompletionFailedError`.
|
|
21
|
+
*/
|
|
22
|
+
export declare function isAddonCancelledError(err: unknown): boolean;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helper for detecting the llama.cpp addon's `Cancelled` status error
|
|
3
|
+
* (`LlmErrors.hpp::Cancelled = 27`) on the continuous-batching path.
|
|
4
|
+
*
|
|
5
|
+
* When a batch is cancelled while it still has prompts queued behind the
|
|
6
|
+
* `parallel` slot limit, `ContinuousBatchScheduler::cancelPendingLocked`
|
|
7
|
+
* fails the whole batch group with a `Cancelled` `StatusError`, which
|
|
8
|
+
* `processBatch` rethrows — so `model.run(prompts)` rejects rather than
|
|
9
|
+
* resolving with partial outputs (see
|
|
10
|
+
* `packages/llm-llamacpp/docs/continuous-batching.md` "Cancellation
|
|
11
|
+
* semantics"). In-flight slots are still cancelled gracefully, but the
|
|
12
|
+
* shared group carries the error.
|
|
13
|
+
*
|
|
14
|
+
* The addon's JS-facing throw goes through `js_throw_error(env, code,
|
|
15
|
+
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
16
|
+
* `qvac_errors::StatusError::codeString()` and formats as
|
|
17
|
+
* `"[ <addonId> :: Cancelled ]"`. This helper lets the batch handler
|
|
18
|
+
* recognise that rejection so it can complete the run with cancelled
|
|
19
|
+
* terminal events (matching the graceful cancel semantics) instead
|
|
20
|
+
* of surfacing a generic `CompletionFailedError`.
|
|
21
|
+
*/
|
|
22
|
+
export function isAddonCancelledError(err) {
|
|
23
|
+
if (typeof err !== 'object' || err === null)
|
|
24
|
+
return false;
|
|
25
|
+
const code = err.code;
|
|
26
|
+
// Anchor to the codeString tail (`[ <addonId> :: Cancelled ]`) so a
|
|
27
|
+
// hypothetical sibling like `CancelledByPolicy` after an addon-side
|
|
28
|
+
// rename doesn't false-positive here.
|
|
29
|
+
return typeof code === 'string' && /::\s*Cancelled\s*\]/.test(code);
|
|
30
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { BatchCompletionStreamPrompt, CompletionStats } from '../../../../schemas/index.js';
|
|
3
|
+
import type { DisposableScope } from '../../../../runtime/disposable-scope.js';
|
|
4
|
+
import type { Logger } from '../../../../logging/types.js';
|
|
5
|
+
type BatchModelEvent = {
|
|
6
|
+
type: 'ids';
|
|
7
|
+
ids: string[];
|
|
8
|
+
} | {
|
|
9
|
+
type: 'token';
|
|
10
|
+
id: string;
|
|
11
|
+
token: string;
|
|
12
|
+
};
|
|
13
|
+
type BatchModelResult = {
|
|
14
|
+
id: string;
|
|
15
|
+
output: string;
|
|
16
|
+
};
|
|
17
|
+
type BatchModelStreamResult = {
|
|
18
|
+
ids: string[];
|
|
19
|
+
results: BatchModelResult[];
|
|
20
|
+
modelExecutionMs: number;
|
|
21
|
+
stats?: CompletionStats;
|
|
22
|
+
};
|
|
23
|
+
export declare function batchCompletion(params: {
|
|
24
|
+
modelId: string;
|
|
25
|
+
prompts: BatchCompletionStreamPrompt[];
|
|
26
|
+
}, opts: {
|
|
27
|
+
signal: AbortSignal;
|
|
28
|
+
scope: DisposableScope;
|
|
29
|
+
logger?: Logger;
|
|
30
|
+
}): AsyncGenerator<BatchModelEvent, BatchModelStreamResult, unknown>;
|
|
31
|
+
export {};
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { TOOLS_MODE } from '../../../../schemas/tools.js';
|
|
2
|
+
import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
|
|
3
|
+
import { getEngineLogger } from '../../../../logging/index.js';
|
|
4
|
+
import { nowMs } from '../../../../profiling/index.js';
|
|
5
|
+
import { buildStreamResult } from '../../../../profiling/model-execution.js';
|
|
6
|
+
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
7
|
+
import { transformMessages } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stream.js';
|
|
8
|
+
import { normalizeCompletionStats } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
|
|
9
|
+
import { appendToolsToHistory, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
10
|
+
const logger = getEngineLogger();
|
|
11
|
+
function runBatchModel(model, prompts) {
|
|
12
|
+
const run = model.run.bind(model);
|
|
13
|
+
return run(prompts);
|
|
14
|
+
}
|
|
15
|
+
function mergeGenerationParams(generationParams, responseFormat) {
|
|
16
|
+
if (!responseFormat)
|
|
17
|
+
return generationParams;
|
|
18
|
+
const jsonSchema = getResponseFormatJsonSchema(responseFormat);
|
|
19
|
+
if (jsonSchema === undefined)
|
|
20
|
+
return generationParams;
|
|
21
|
+
return {
|
|
22
|
+
...(generationParams ?? {}),
|
|
23
|
+
json_schema: jsonSchema
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
function renderPromptHistory(prompt, options) {
|
|
27
|
+
const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0 ? prompt.tools : undefined;
|
|
28
|
+
let historyWithTools = prompt.history;
|
|
29
|
+
if (tools) {
|
|
30
|
+
historyWithTools =
|
|
31
|
+
options.toolsMode === TOOLS_MODE.dynamic
|
|
32
|
+
? appendToolsToHistory(prompt.history, tools)
|
|
33
|
+
: prependToolsToHistory(prompt.history, tools);
|
|
34
|
+
}
|
|
35
|
+
// Uses the same attachment expansion as single completion: each
|
|
36
|
+
// attachment becomes an addon `type: "media"` message before the text turn.
|
|
37
|
+
return transformMessages(historyWithTools);
|
|
38
|
+
}
|
|
39
|
+
function buildBatchPrompt(prompt, options) {
|
|
40
|
+
const mergedGenerationParams = mergeGenerationParams(prompt.generationParams, prompt.responseFormat);
|
|
41
|
+
return {
|
|
42
|
+
...(prompt.id !== undefined && { id: prompt.id }),
|
|
43
|
+
prompt: renderPromptHistory(prompt, options),
|
|
44
|
+
...(mergedGenerationParams && {
|
|
45
|
+
runOptions: { generationParams: mergedGenerationParams }
|
|
46
|
+
})
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
function isBatchOutputChunk(output) {
|
|
50
|
+
return (output !== null &&
|
|
51
|
+
typeof output === 'object' &&
|
|
52
|
+
'id' in output &&
|
|
53
|
+
'chunk' in output &&
|
|
54
|
+
typeof output.id === 'string' &&
|
|
55
|
+
typeof output.chunk === 'string');
|
|
56
|
+
}
|
|
57
|
+
export async function* batchCompletion(params, opts) {
|
|
58
|
+
const { modelId, prompts } = params;
|
|
59
|
+
const { signal, scope } = opts;
|
|
60
|
+
const requestLogger = opts.logger ?? logger;
|
|
61
|
+
const model = getModel(modelId);
|
|
62
|
+
const modelConfig = getModelConfig(modelId);
|
|
63
|
+
const renderOptions = {
|
|
64
|
+
toolsEnabled: modelConfig.tools === true,
|
|
65
|
+
toolsMode: modelConfig.toolsMode
|
|
66
|
+
};
|
|
67
|
+
const onAbort = () => {
|
|
68
|
+
const addon = model.addon;
|
|
69
|
+
if (addon?.cancel) {
|
|
70
|
+
addon.cancel.call(addon).catch((err) => {
|
|
71
|
+
requestLogger.warn(`[cancel] addon.cancel() rejected during batch abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
76
|
+
if (signal.aborted)
|
|
77
|
+
onAbort();
|
|
78
|
+
scope.defer(() => {
|
|
79
|
+
signal.removeEventListener('abort', onAbort);
|
|
80
|
+
});
|
|
81
|
+
const addonPrompts = prompts.map((prompt) => buildBatchPrompt(prompt, renderOptions));
|
|
82
|
+
const modelStart = nowMs();
|
|
83
|
+
const response = await runBatchModel(model, addonPrompts);
|
|
84
|
+
const ids = response.ids;
|
|
85
|
+
yield { type: 'ids', ids };
|
|
86
|
+
for await (const output of response.iterate()) {
|
|
87
|
+
if (!isBatchOutputChunk(output))
|
|
88
|
+
continue;
|
|
89
|
+
yield { type: 'token', id: output.id, token: output.chunk };
|
|
90
|
+
}
|
|
91
|
+
const results = await response.await();
|
|
92
|
+
const modelExecutionMs = nowMs() - modelStart;
|
|
93
|
+
const stats = normalizeCompletionStats(response.stats);
|
|
94
|
+
return {
|
|
95
|
+
...buildStreamResult(modelExecutionMs, stats),
|
|
96
|
+
ids,
|
|
97
|
+
results
|
|
98
|
+
};
|
|
99
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export declare function logCacheStatus(cacheKey: string, isReusing: boolean): void;
|
|
2
|
+
export declare function logCacheInit(cacheKey: string, systemPrompt: string, toolCount: number): void;
|
|
3
|
+
export declare function logMessagesToAddon(messages: unknown[], phase?: 'CACHE_INIT' | 'PROMPT_SEND' | 'NO_CACHE'): void;
|
|
4
|
+
export declare function logCacheDisabled(): void;
|
|
5
|
+
export declare function logCacheSave(sessionPath: string): void;
|
|
6
|
+
export declare function logCacheSaveError(sessionPath: string, err: unknown): void;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { getEngineLogger } from '../../../../logging/index.js';
|
|
2
|
+
const logger = getEngineLogger();
|
|
3
|
+
function formatMessages(messages) {
|
|
4
|
+
return messages
|
|
5
|
+
.map((m) => {
|
|
6
|
+
const role = m.role || m.type || '?';
|
|
7
|
+
const content = m.content
|
|
8
|
+
? m.content.length > 40
|
|
9
|
+
? m.content.substring(0, 40) + '...'
|
|
10
|
+
: m.content
|
|
11
|
+
: m.name || '(no content)';
|
|
12
|
+
return `{${role}: "${content}"}`;
|
|
13
|
+
})
|
|
14
|
+
.join(', ');
|
|
15
|
+
}
|
|
16
|
+
export function logCacheStatus(cacheKey, isReusing) {
|
|
17
|
+
const status = isReusing ? 'REUSING' : 'CREATING';
|
|
18
|
+
logger.debug(`[kv-cache] [${cacheKey}] ${status} cache`);
|
|
19
|
+
}
|
|
20
|
+
export function logCacheInit(cacheKey, systemPrompt, toolCount) {
|
|
21
|
+
const promptLen = systemPrompt.length;
|
|
22
|
+
logger.debug(`[kv-cache] [${cacheKey}] Initializing cache (prompt: ${promptLen} chars, tools: ${toolCount})`);
|
|
23
|
+
}
|
|
24
|
+
export function logMessagesToAddon(messages, phase = 'PROMPT_SEND') {
|
|
25
|
+
const typedMessages = messages;
|
|
26
|
+
logger.debug(`[kv-cache] [${phase}] Sending ${typedMessages.length} msg(s): [${formatMessages(typedMessages)}]`);
|
|
27
|
+
}
|
|
28
|
+
export function logCacheDisabled() {
|
|
29
|
+
logger.debug('[kv-cache] Cache disabled');
|
|
30
|
+
}
|
|
31
|
+
export function logCacheSave(sessionPath) {
|
|
32
|
+
logger.debug(`[kv-cache] Saving session: ...${sessionPath.slice(-20)}`);
|
|
33
|
+
}
|
|
34
|
+
export function logCacheSaveError(sessionPath, err) {
|
|
35
|
+
logger.warn(`[kv-cache] Failed to save session: ...${sessionPath.slice(-20)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
36
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { CompletionStats } from '../../../../schemas/index.js';
|
|
2
|
+
import type { LlmStats } from '../../../../utils/addon-responses.js';
|
|
3
|
+
export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
|
|
4
|
+
timeToFirstToken?: number | undefined;
|
|
5
|
+
tokensPerSecond?: number | undefined;
|
|
6
|
+
cacheTokens?: number | undefined;
|
|
7
|
+
promptTokens?: number | undefined;
|
|
8
|
+
generatedTokens?: number | undefined;
|
|
9
|
+
emittedTokens?: number | undefined;
|
|
10
|
+
avgConcurrentSeq?: number | undefined;
|
|
11
|
+
backendDevice?: "gpu" | "cpu" | undefined;
|
|
12
|
+
} | undefined;
|
|
13
|
+
/**
|
|
14
|
+
* Attach the count of non-empty pieces the addon streamed without
|
|
15
|
+
* overwriting `generatedTokens` (`llama_perf` `n_eval`). Length / KV-cache
|
|
16
|
+
* decisions keep the decode count; usage reporting prefers `emittedTokens`.
|
|
17
|
+
* Zero is attached explicitly so inflated `n_eval` cannot be used as usage
|
|
18
|
+
* when nothing was streamed.
|
|
19
|
+
*/
|
|
20
|
+
export declare function withEmittedTokens(stats: CompletionStats | undefined, emittedPieces: number): CompletionStats | undefined;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
function finiteNumber(value) {
|
|
2
|
+
return value !== undefined && Number.isFinite(value) ? value : undefined;
|
|
3
|
+
}
|
|
4
|
+
export function normalizeCompletionStats(stats) {
|
|
5
|
+
if (!stats)
|
|
6
|
+
return undefined;
|
|
7
|
+
const timeToFirstToken = finiteNumber(stats.TTFT);
|
|
8
|
+
const tokensPerSecond = finiteNumber(stats.TPS);
|
|
9
|
+
const cacheTokens = finiteNumber(stats.CacheTokens);
|
|
10
|
+
const promptTokens = finiteNumber(stats.promptTokens);
|
|
11
|
+
const generatedTokens = finiteNumber(stats.generatedTokens);
|
|
12
|
+
const avgConcurrentSeq = finiteNumber(stats.avgConcurrentSeq);
|
|
13
|
+
const normalized = {
|
|
14
|
+
...(timeToFirstToken !== undefined && { timeToFirstToken }),
|
|
15
|
+
...(tokensPerSecond !== undefined && { tokensPerSecond }),
|
|
16
|
+
...(cacheTokens !== undefined && { cacheTokens }),
|
|
17
|
+
...(promptTokens !== undefined && { promptTokens }),
|
|
18
|
+
...(generatedTokens !== undefined && { generatedTokens }),
|
|
19
|
+
...(avgConcurrentSeq !== undefined && { avgConcurrentSeq }),
|
|
20
|
+
...(stats.backendDevice !== undefined && { backendDevice: stats.backendDevice })
|
|
21
|
+
};
|
|
22
|
+
if (timeToFirstToken === undefined &&
|
|
23
|
+
tokensPerSecond === undefined &&
|
|
24
|
+
cacheTokens === undefined &&
|
|
25
|
+
promptTokens === undefined &&
|
|
26
|
+
generatedTokens === undefined &&
|
|
27
|
+
avgConcurrentSeq === undefined &&
|
|
28
|
+
stats.backendDevice === undefined) {
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
return normalized;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Attach the count of non-empty pieces the addon streamed without
|
|
35
|
+
* overwriting `generatedTokens` (`llama_perf` `n_eval`). Length / KV-cache
|
|
36
|
+
* decisions keep the decode count; usage reporting prefers `emittedTokens`.
|
|
37
|
+
* Zero is attached explicitly so inflated `n_eval` cannot be used as usage
|
|
38
|
+
* when nothing was streamed.
|
|
39
|
+
*/
|
|
40
|
+
export function withEmittedTokens(stats, emittedPieces) {
|
|
41
|
+
const emittedTokens = emittedPieces;
|
|
42
|
+
if (!stats && emittedTokens === 0)
|
|
43
|
+
return undefined;
|
|
44
|
+
return {
|
|
45
|
+
...(stats ?? {}),
|
|
46
|
+
emittedTokens
|
|
47
|
+
};
|
|
48
|
+
}
|