@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -50,11 +50,13 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import { requestParent } from '../../schemas/plugin.js';
|
|
53
54
|
import { getModel, getModelConfig, getModelEntry } from '../../runtime/model-registry.js';
|
|
54
55
|
import { ModelType } from '../../schemas/index.js';
|
|
55
56
|
import { createAudioStream } from '../../utils/audio-input.js';
|
|
57
|
+
import { untilAborted } from '../../utils/until-aborted.js';
|
|
56
58
|
import { getEngineLogger } from '../../logging/index.js';
|
|
57
|
-
import { TranscriptionFailedError } from '../../errors/index.js';
|
|
59
|
+
import { InferenceCancelledError, TranscriptionFailedError } from '../../errors/index.js';
|
|
58
60
|
import { nowMs } from '../../profiling/index.js';
|
|
59
61
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
60
62
|
import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
|
|
@@ -64,6 +66,9 @@ import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
|
64
66
|
import { isEndOfTurnEvent, isVadEvent, toEndOfTurnEvent, toVadStateEvent } from '../../utils/asr-events.js';
|
|
65
67
|
import { buildWhisperReloadConfig } from '../../plugins/builtin/asr-ggml/config.js';
|
|
66
68
|
export { assertMetadataSupported, toTranscribeSegment };
|
|
69
|
+
async function* iterateAll(source) {
|
|
70
|
+
yield* source;
|
|
71
|
+
}
|
|
67
72
|
function hasRunStreaming(model) {
|
|
68
73
|
return 'runStreaming' in model && typeof model.runStreaming === 'function';
|
|
69
74
|
}
|
|
@@ -173,10 +178,23 @@ function mapTranscribeStats(stats) {
|
|
|
173
178
|
}),
|
|
174
179
|
...(stats?.encoderUsedCoreml !== undefined && {
|
|
175
180
|
encoderUsedCoreml: stats.encoderUsedCoreml
|
|
181
|
+
}),
|
|
182
|
+
...(stats?.aoscActive !== undefined && { aoscActive: stats.aoscActive }),
|
|
183
|
+
// MOSS-Transcribe-Diarize reports its counters under its own names. Its
|
|
184
|
+
// stage timings land on the shared encoder/decoder fields and its segment
|
|
185
|
+
// count on `totalSegments`; the token counts and prefill are MOSS-only.
|
|
186
|
+
...(stats?.segments !== undefined && { totalSegments: stats.segments }),
|
|
187
|
+
...(stats?.encodeMs !== undefined && { encoderTime: stats.encodeMs }),
|
|
188
|
+
...(stats?.decodeMs !== undefined && { decoderTime: stats.decodeMs }),
|
|
189
|
+
...(stats?.prefillMs !== undefined && { prefillTime: stats.prefillMs }),
|
|
190
|
+
...(stats?.audioTokens !== undefined && { audioTokens: stats.audioTokens }),
|
|
191
|
+
...(stats?.promptTokens !== undefined && { promptTokens: stats.promptTokens }),
|
|
192
|
+
...(stats?.generatedTokens !== undefined && {
|
|
193
|
+
generatedTokens: stats.generatedTokens
|
|
176
194
|
})
|
|
177
195
|
};
|
|
178
196
|
}
|
|
179
|
-
export async function* transcribe(params, requestId) {
|
|
197
|
+
export async function* transcribe(params, requestId, sessionId, context) {
|
|
180
198
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
181
199
|
try {
|
|
182
200
|
const { modelId, metadata } = params;
|
|
@@ -186,9 +204,18 @@ export async function* transcribe(params, requestId) {
|
|
|
186
204
|
// caller didn't send one.
|
|
187
205
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
188
206
|
requestId: requestId ?? generateRandomRequestId(),
|
|
207
|
+
sessionId,
|
|
208
|
+
...requestParent(context),
|
|
189
209
|
kind: 'transcribe',
|
|
190
210
|
modelId
|
|
191
|
-
})
|
|
211
|
+
})
|
|
212
|
+
// A request cancelled while it queued holds no slot; the model-wide
|
|
213
|
+
// `addon.cancel()` below would stop the job that does.
|
|
214
|
+
, true);
|
|
215
|
+
// A request cancelled while it queued holds no slot; the model-wide
|
|
216
|
+
// `addon.cancel()` below would stop the job that does.
|
|
217
|
+
if (ctx.signal.aborted)
|
|
218
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
192
219
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
193
220
|
const engineType = getEngineModelType(modelId);
|
|
194
221
|
const hasMossOptions = params.hotwords !== undefined || params.maxNewTokens !== undefined;
|
|
@@ -248,9 +275,7 @@ export async function* transcribe(params, requestId) {
|
|
|
248
275
|
const chunks = (Array.isArray(output) ? output : [output]);
|
|
249
276
|
if (metadata) {
|
|
250
277
|
for (const chunk of chunks) {
|
|
251
|
-
if (!chunk
|
|
252
|
-
continue;
|
|
253
|
-
if (silenceMarker && chunk.text.includes(silenceMarker))
|
|
278
|
+
if (!carriesSegment(chunk, silenceMarker))
|
|
254
279
|
continue;
|
|
255
280
|
yield toTranscribeSegment(chunk);
|
|
256
281
|
}
|
|
@@ -297,7 +322,7 @@ function buildRunStreamingOpts(engineType, opts) {
|
|
|
297
322
|
}
|
|
298
323
|
return runOpts;
|
|
299
324
|
}
|
|
300
|
-
export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId) {
|
|
325
|
+
export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId, sessionId, context) {
|
|
301
326
|
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
302
327
|
try {
|
|
303
328
|
// Same `kind: "transcribe"` as the unary variant — the registry
|
|
@@ -306,9 +331,13 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
306
331
|
// either shape.
|
|
307
332
|
const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
|
|
308
333
|
requestId: requestId ?? generateRandomRequestId(),
|
|
334
|
+
sessionId,
|
|
335
|
+
...requestParent(context),
|
|
309
336
|
kind: 'transcribe',
|
|
310
337
|
modelId
|
|
311
338
|
}), true);
|
|
339
|
+
if (ctx.signal.aborted)
|
|
340
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
312
341
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
313
342
|
const engineType = getEngineModelType(modelId);
|
|
314
343
|
assertMetadataSupported(modelId, engineType, metadata);
|
|
@@ -321,26 +350,19 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
321
350
|
if (!hasRunStreaming(model)) {
|
|
322
351
|
throw new TranscriptionFailedError(`Model ${modelId} does not support streaming transcription`);
|
|
323
352
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
};
|
|
332
|
-
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
333
|
-
if (ctx.signal.aborted)
|
|
334
|
-
onAbort();
|
|
335
|
-
ctx.scope.defer(() => {
|
|
336
|
-
ctx.signal.removeEventListener('abort', onAbort);
|
|
337
|
-
});
|
|
353
|
+
// A streaming session ends when its input ends, so an abort ends the input
|
|
354
|
+
// and the session drains the audio it already has, yielding nothing more.
|
|
355
|
+
// The model-wide `addon.cancel()` the unary path uses is not safe here: a
|
|
356
|
+
// parakeet stream cancelled that way leaves its stream state to the next
|
|
357
|
+
// session, whose output starts with the cancelled session's audio and never
|
|
358
|
+
// ends.
|
|
359
|
+
const input = untilAborted(iterateAll(audioInputStream), ctx.signal);
|
|
338
360
|
const runOpts = buildRunStreamingOpts(engineType, opts);
|
|
339
361
|
const modelStart = nowMs();
|
|
340
|
-
const response = await model.runStreaming(
|
|
362
|
+
const response = await model.runStreaming(input, runOpts);
|
|
341
363
|
for await (const output of response.iterate()) {
|
|
342
364
|
if (ctx.signal.aborted)
|
|
343
|
-
|
|
365
|
+
continue;
|
|
344
366
|
requestLogger.debug('Live Transcription Update:', output);
|
|
345
367
|
if (!Array.isArray(output)) {
|
|
346
368
|
if (isVadEvent(output)) {
|
|
@@ -386,17 +408,30 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
386
408
|
await result_2;
|
|
387
409
|
}
|
|
388
410
|
}
|
|
411
|
+
/**
|
|
412
|
+
* Whether a native segment becomes a metadata segment. Text-less segments are
|
|
413
|
+
* dropped, except an explicit final marker: parakeet streaming closes a chunk
|
|
414
|
+
* with `isFinal: true` and no text, and that marker is the only finality signal
|
|
415
|
+
* the caller gets for the chunk.
|
|
416
|
+
*/
|
|
417
|
+
function carriesSegment(segment, silenceMarker) {
|
|
418
|
+
if (!segment.text)
|
|
419
|
+
return segment.isFinal === true;
|
|
420
|
+
return !silenceMarker || !segment.text.includes(silenceMarker);
|
|
421
|
+
}
|
|
389
422
|
function* emitSegment(segment, metadata, silenceMarker) {
|
|
423
|
+
if (metadata) {
|
|
424
|
+
if (carriesSegment(segment, silenceMarker))
|
|
425
|
+
yield toTranscribeSegment(segment);
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
390
428
|
if (!segment.text) {
|
|
391
429
|
return;
|
|
392
430
|
}
|
|
393
431
|
if (silenceMarker && segment.text.includes(silenceMarker)) {
|
|
394
432
|
return;
|
|
395
433
|
}
|
|
396
|
-
if (
|
|
397
|
-
yield toTranscribeSegment(segment);
|
|
398
|
-
}
|
|
399
|
-
else if (segment.text.trim()) {
|
|
434
|
+
if (segment.text.trim()) {
|
|
400
435
|
yield segment.text;
|
|
401
436
|
}
|
|
402
437
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../schemas/plugin.js';
|
|
1
2
|
import { type TranslateParams, type TranslationStats } from '../../schemas/index.js';
|
|
2
3
|
export declare function getLanguage(code: string | undefined): string;
|
|
3
4
|
export declare function isAfrican(code: string | undefined): boolean;
|
|
4
|
-
export declare function translate(params: TranslateParams, requestId?: string): AsyncGenerator<string, {
|
|
5
|
+
export declare function translate(params: TranslateParams, requestId?: string, sessionId?: string, invoke?: PluginHandlerContext): AsyncGenerator<string, {
|
|
5
6
|
modelExecutionMs: number;
|
|
6
7
|
stats?: TranslationStats;
|
|
7
8
|
}, unknown>;
|
|
@@ -50,6 +50,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import { requestParent } from '../../schemas/plugin.js';
|
|
53
54
|
import { getModelEntry } from '../../runtime/model-registry.js';
|
|
54
55
|
import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LANGUAGES_MAP } from '../../schemas/index.js';
|
|
55
56
|
import { getLangName, detectOne } from '@qvac/langdetect-text';
|
|
@@ -122,7 +123,7 @@ const LLM_TRANSLATE_GENERATION_PARAMS = {
|
|
|
122
123
|
function shouldSkipPerCallSampling(modelName) {
|
|
123
124
|
return !!modelName && modelName.startsWith('AFRICAN_');
|
|
124
125
|
}
|
|
125
|
-
export async function* translate(params, requestId) {
|
|
126
|
+
export async function* translate(params, requestId, sessionId, invoke) {
|
|
126
127
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
127
128
|
try {
|
|
128
129
|
const { modelId, text, modelType: inputModelType } = params;
|
|
@@ -168,6 +169,8 @@ export async function* translate(params, requestId) {
|
|
|
168
169
|
// fine because the result is dropped either way.
|
|
169
170
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
170
171
|
requestId: requestId ?? generateRandomRequestId(),
|
|
172
|
+
sessionId,
|
|
173
|
+
...requestParent(invoke),
|
|
171
174
|
kind: 'translate',
|
|
172
175
|
modelId,
|
|
173
176
|
// LLM translate joins the completion lane (its cap = the model's own
|
|
@@ -1,2 +1,18 @@
|
|
|
1
1
|
import { type UnloadModelParams } from '../../schemas/index.js';
|
|
2
|
-
export
|
|
2
|
+
export interface UnloadModelResult {
|
|
3
|
+
/** The caller's lease was dropped. False when the caller held none. */
|
|
4
|
+
released: boolean;
|
|
5
|
+
/** The model left memory. False while another session still holds a lease. */
|
|
6
|
+
unloaded: boolean;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Releases the caller session's lease on the model. The model is torn down
|
|
10
|
+
* only when no lease remains; a session that never loaded the model cannot
|
|
11
|
+
* unload it from under the sessions that did.
|
|
12
|
+
*
|
|
13
|
+
* Runs under the model's load lock: a load of the same configuration that
|
|
14
|
+
* arrives during the teardown waits for it and then loads a fresh copy,
|
|
15
|
+
* instead of finding the id unregistered and loading a second native model
|
|
16
|
+
* while the first is still being freed.
|
|
17
|
+
*/
|
|
18
|
+
export declare function unloadModel(params: UnloadModelParams): Promise<UnloadModelResult>;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
|
-
import { unregisterModel } from '../../runtime/model-registry.js';
|
|
3
|
+
import { getModelEntry, releaseModelLease, unregisterModel } from '../../runtime/model-registry.js';
|
|
4
4
|
import { getRequestRegistry } from '../../runtime/index.js';
|
|
5
|
+
import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
|
|
5
6
|
import { unregisterAllLoggingStreams } from '../../runtime/logging-stream-registry.js';
|
|
6
7
|
import { clearFinetuneRuntimeState } from '../../plugins/builtin/llamacpp-completion/ops/finetune.js';
|
|
7
8
|
import { unregisterAddonLogger, getEngineLogger } from '../../logging/index.js';
|
|
@@ -9,9 +10,36 @@ import { unloadModelParamsSchema } from '../../schemas/index.js';
|
|
|
9
10
|
import { ModelNotLoadedError } from '../../errors/index.js';
|
|
10
11
|
import { detectShardedModel } from '../../utils/index.js';
|
|
11
12
|
import { getClearStorageTarget } from '../../utils/cache/paths.js';
|
|
13
|
+
import { withModelWorkLock } from '../../runtime/model-work-lock.js';
|
|
12
14
|
const logger = getEngineLogger();
|
|
15
|
+
/**
|
|
16
|
+
* Releases the caller session's lease on the model. The model is torn down
|
|
17
|
+
* only when no lease remains; a session that never loaded the model cannot
|
|
18
|
+
* unload it from under the sessions that did.
|
|
19
|
+
*
|
|
20
|
+
* Runs under the model's load lock: a load of the same configuration that
|
|
21
|
+
* arrives during the teardown waits for it and then loads a fresh copy,
|
|
22
|
+
* instead of finding the id unregistered and loading a second native model
|
|
23
|
+
* while the first is still being freed.
|
|
24
|
+
*/
|
|
13
25
|
export async function unloadModel(params) {
|
|
14
|
-
const
|
|
26
|
+
const parsed = unloadModelParamsSchema.parse(params);
|
|
27
|
+
return withModelWorkLock(parsed.modelId, () => unloadModelLocked(parsed));
|
|
28
|
+
}
|
|
29
|
+
async function unloadModelLocked(params) {
|
|
30
|
+
const { modelId, clearStorage, sessionId = OWNER_SESSION_ID } = params;
|
|
31
|
+
if (!getModelEntry(modelId)) {
|
|
32
|
+
throw new ModelNotLoadedError(modelId);
|
|
33
|
+
}
|
|
34
|
+
const remaining = releaseModelLease(modelId, sessionId);
|
|
35
|
+
if (remaining === null) {
|
|
36
|
+
logger.info(`Session ${sessionId} holds no lease on model ${modelId}; nothing released`);
|
|
37
|
+
return { released: false, unloaded: false };
|
|
38
|
+
}
|
|
39
|
+
if (remaining > 0) {
|
|
40
|
+
logger.info(`Model ${modelId} lease released by ${sessionId}; ${remaining} lease(s) remain`);
|
|
41
|
+
return { released: true, unloaded: false };
|
|
42
|
+
}
|
|
15
43
|
const entry = unregisterModel(modelId);
|
|
16
44
|
if (!entry) {
|
|
17
45
|
throw new ModelNotLoadedError(modelId);
|
|
@@ -46,4 +74,5 @@ export async function unloadModel(params) {
|
|
|
46
74
|
unregisterAllLoggingStreams(modelId);
|
|
47
75
|
logger.info(`Model ${modelId} unloaded`);
|
|
48
76
|
});
|
|
77
|
+
return { released: true, unloaded: true };
|
|
49
78
|
}
|
|
@@ -11,7 +11,7 @@ export declare function storageFromBitWidth(bitWidth: unknown): VectorIndexStora
|
|
|
11
11
|
* The vector index feature's only backend: the fabric vector index that the
|
|
12
12
|
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
13
13
|
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
14
|
-
* dimension rule (multiple of 8, at most
|
|
14
|
+
* dimension rule (multiple of 8, at most 16384), which the native index
|
|
15
15
|
* enforces itself.
|
|
16
16
|
*/
|
|
17
17
|
export declare function getTurboVecBackend(): VectorIndexBackend | undefined;
|
|
@@ -48,6 +48,21 @@ function openIndex(index, storage) {
|
|
|
48
48
|
return index.length;
|
|
49
49
|
},
|
|
50
50
|
storage,
|
|
51
|
+
get calibrated() {
|
|
52
|
+
return index.calibrated;
|
|
53
|
+
},
|
|
54
|
+
get stagedSearch() {
|
|
55
|
+
return index.stagedSearch;
|
|
56
|
+
},
|
|
57
|
+
setStagedSearch(enabled) {
|
|
58
|
+
// Without the getter the setting could not be confirmed, so a provider
|
|
59
|
+
// needs both halves of the control.
|
|
60
|
+
if (!index.setStagedSearch || typeof index.stagedSearch !== 'boolean') {
|
|
61
|
+
throw new Error('The vector index provider does not support staged search controls');
|
|
62
|
+
}
|
|
63
|
+
index.setStagedSearch(enabled);
|
|
64
|
+
return index.stagedSearch;
|
|
65
|
+
},
|
|
51
66
|
add(vectors, ids) {
|
|
52
67
|
index.addWithIds(vectors, ids);
|
|
53
68
|
needsPrepare = true;
|
|
@@ -80,7 +95,7 @@ function openIndex(index, storage) {
|
|
|
80
95
|
* The vector index feature's only backend: the fabric vector index that the
|
|
81
96
|
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
82
97
|
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
83
|
-
* dimension rule (multiple of 8, at most
|
|
98
|
+
* dimension rule (multiple of 8, at most 16384), which the native index
|
|
84
99
|
* enforces itself.
|
|
85
100
|
*/
|
|
86
101
|
export function getTurboVecBackend() {
|
|
@@ -54,7 +54,8 @@ export function buildOperationEvent(op, profileId, ts, executionMs, request, fin
|
|
|
54
54
|
Object.assign(gauges, extracted);
|
|
55
55
|
}
|
|
56
56
|
const requestTags = config.getTags?.(request);
|
|
57
|
-
|
|
57
|
+
// A handler that threw has no response; its event carries request tags only.
|
|
58
|
+
const resultTags = finalResponse ? config.getTagsFromResult?.(finalResponse) : undefined;
|
|
58
59
|
const tags = { ...requestTags, ...resultTags };
|
|
59
60
|
const hasGauges = Object.keys(gauges).length > 0;
|
|
60
61
|
const hasTags = Object.keys(tags).length > 0;
|
|
@@ -257,25 +258,27 @@ registerOperationMetrics({
|
|
|
257
258
|
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
258
259
|
}
|
|
259
260
|
});
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
261
|
+
for (const op of ['embed', 'embedBatch']) {
|
|
262
|
+
registerOperationMetrics({
|
|
263
|
+
op,
|
|
264
|
+
kind: 'handler',
|
|
265
|
+
getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
|
|
266
|
+
fromResult: (res) => {
|
|
267
|
+
const gauges = {};
|
|
268
|
+
const modelExecMs = readModelExecutionMs(res);
|
|
269
|
+
if (modelExecMs !== undefined)
|
|
270
|
+
gauges['modelExecutionTime'] = modelExecMs;
|
|
271
|
+
if (res.stats?.totalTime !== undefined)
|
|
272
|
+
gauges['totalTime'] = res.stats.totalTime;
|
|
273
|
+
if (res.stats?.tokensPerSecond !== undefined) {
|
|
274
|
+
gauges['tokensPerSecond'] = res.stats.tokensPerSecond;
|
|
275
|
+
}
|
|
276
|
+
if (res.stats?.totalTokens !== undefined)
|
|
277
|
+
gauges['totalTokens'] = res.stats.totalTokens;
|
|
278
|
+
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
273
279
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
277
|
-
}
|
|
278
|
-
});
|
|
280
|
+
});
|
|
281
|
+
}
|
|
279
282
|
registerOperationMetrics({
|
|
280
283
|
op: 'ocrStream',
|
|
281
284
|
kind: 'handler',
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type EmbeddingFunction } from '@qvac/rag';
|
|
2
|
+
import type { PluginHandlerContext } from '../schemas/plugin.js';
|
|
3
|
+
export declare function setRagEmbedParent(workspace: string | undefined, parent: PluginHandlerContext): void;
|
|
4
|
+
/** Clears the parent only while it is still the one set, so an older operation's unwind never drops a newer one. */
|
|
5
|
+
export declare function clearRagEmbedParent(workspace: string | undefined, parent: PluginHandlerContext): void;
|
|
6
|
+
export declare function getRagEmbedParent(workspace: string | undefined): PluginHandlerContext | undefined;
|
|
7
|
+
/**
|
|
8
|
+
* The text the workspace embedding function is asked for when a search runs:
|
|
9
|
+
* `@qvac/rag` trims a query before it embeds it, so a search embeds and
|
|
10
|
+
* stashes the trimmed text to be found by the same key.
|
|
11
|
+
*/
|
|
12
|
+
export declare function queryEmbeddingText(query: string): string;
|
|
13
|
+
export declare function stashQueryEmbedding(workspace: string | undefined, query: string, vector: number[]): void;
|
|
14
|
+
export declare function takeQueryEmbedding(workspace: string | undefined, query: string): number[] | undefined;
|
|
15
|
+
/** The embedding function a workspace's RAG instance is created with. */
|
|
16
|
+
export declare function createWorkspaceEmbeddingFunction(modelId: string, workspace: string | undefined): EmbeddingFunction;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { ERR_CODES, QvacErrorRAG } from '@qvac/rag';
|
|
2
|
+
import { InferenceCancelledError } from '../errors/index.js';
|
|
3
|
+
import { embed } from '../plugins/ops/embed.js';
|
|
4
|
+
import { getWorkspaceKey } from '../rag/rag-operation-manager.js';
|
|
5
|
+
/**
|
|
6
|
+
* The request a workspace's embeddings run under.
|
|
7
|
+
*
|
|
8
|
+
* A workspace holds one RAG instance with one embedding function for its
|
|
9
|
+
* lifetime, so the function cannot be bound to the request that is using it.
|
|
10
|
+
* Instead, the registered RAG operation on the workspace (`ingest`,
|
|
11
|
+
* `saveEmbeddings`, `reindex`; one at a time, see `beginRagContext`) publishes
|
|
12
|
+
* itself here for its duration, and the embedding function begins every
|
|
13
|
+
* `embed` as a child of it: the embedding runs as the caller's session and
|
|
14
|
+
* aborts with the operation. A `search` is not a registered operation and
|
|
15
|
+
* runs beside them, so it embeds its query itself, under its own request, and
|
|
16
|
+
* leaves the vector here for the function to hand back.
|
|
17
|
+
*/
|
|
18
|
+
const parentByWorkspace = new Map();
|
|
19
|
+
const queryVectorsByWorkspace = new Map();
|
|
20
|
+
export function setRagEmbedParent(workspace, parent) {
|
|
21
|
+
parentByWorkspace.set(getWorkspaceKey(workspace), parent);
|
|
22
|
+
}
|
|
23
|
+
/** Clears the parent only while it is still the one set, so an older operation's unwind never drops a newer one. */
|
|
24
|
+
export function clearRagEmbedParent(workspace, parent) {
|
|
25
|
+
const key = getWorkspaceKey(workspace);
|
|
26
|
+
if (parentByWorkspace.get(key) === parent)
|
|
27
|
+
parentByWorkspace.delete(key);
|
|
28
|
+
}
|
|
29
|
+
export function getRagEmbedParent(workspace) {
|
|
30
|
+
return parentByWorkspace.get(getWorkspaceKey(workspace));
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* The text the workspace embedding function is asked for when a search runs:
|
|
34
|
+
* `@qvac/rag` trims a query before it embeds it, so a search embeds and
|
|
35
|
+
* stashes the trimmed text to be found by the same key.
|
|
36
|
+
*/
|
|
37
|
+
export function queryEmbeddingText(query) {
|
|
38
|
+
return query.trim();
|
|
39
|
+
}
|
|
40
|
+
export function stashQueryEmbedding(workspace, query, vector) {
|
|
41
|
+
const key = getWorkspaceKey(workspace);
|
|
42
|
+
let vectors = queryVectorsByWorkspace.get(key);
|
|
43
|
+
if (!vectors) {
|
|
44
|
+
vectors = new Map();
|
|
45
|
+
queryVectorsByWorkspace.set(key, vectors);
|
|
46
|
+
}
|
|
47
|
+
vectors.set(query, vector);
|
|
48
|
+
}
|
|
49
|
+
export function takeQueryEmbedding(workspace, query) {
|
|
50
|
+
const key = getWorkspaceKey(workspace);
|
|
51
|
+
const vectors = queryVectorsByWorkspace.get(key);
|
|
52
|
+
const vector = vectors?.get(query);
|
|
53
|
+
if (vectors && vector !== undefined) {
|
|
54
|
+
vectors.delete(query);
|
|
55
|
+
if (vectors.size === 0)
|
|
56
|
+
queryVectorsByWorkspace.delete(key);
|
|
57
|
+
}
|
|
58
|
+
return vector;
|
|
59
|
+
}
|
|
60
|
+
/** The embedding function a workspace's RAG instance is created with. */
|
|
61
|
+
export function createWorkspaceEmbeddingFunction(modelId, workspace) {
|
|
62
|
+
return async (text) => {
|
|
63
|
+
if (typeof text === 'string') {
|
|
64
|
+
const stashed = takeQueryEmbedding(workspace, text);
|
|
65
|
+
if (stashed !== undefined)
|
|
66
|
+
return stashed;
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
const result = await embed({ modelId, text }, undefined, undefined, getRagEmbedParent(workspace));
|
|
70
|
+
return result.embedding;
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
// `@qvac/rag` reports any other error from this function as a failed
|
|
74
|
+
// generation, so a cancelled embed is handed back as its cancellation.
|
|
75
|
+
if (error instanceof InferenceCancelledError) {
|
|
76
|
+
throw new QvacErrorRAG({ code: ERR_CODES.OPERATION_CANCELLED, cause: error });
|
|
77
|
+
}
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
}
|
package/dist/rag/handler.d.ts
CHANGED
|
@@ -1,2 +1,27 @@
|
|
|
1
1
|
import type { RagRequest, RagResponse, RagProgressUpdate } from '../schemas/index.js';
|
|
2
|
+
import { type ManagedRequestContext } from '../runtime/index.js';
|
|
3
|
+
/**
|
|
4
|
+
* Begin a registry-tracked RAG context with workspace-level pre-emption.
|
|
5
|
+
*
|
|
6
|
+
* Workspace-level admission lives in the dispatcher rather than as a
|
|
7
|
+
* registry policy primitive (it's a dispatch concern, not a registry
|
|
8
|
+
* `kind` admission rule). The sequence is **cancel-prior → begin-new**:
|
|
9
|
+
* if another RAG operation is already running on the same workspace,
|
|
10
|
+
* cancel it first, then begin the new context. Reversing the order
|
|
11
|
+
* would cancel the just-installed context.
|
|
12
|
+
*
|
|
13
|
+
* The workspace → requestId map is updated after `begin(...)` succeeds
|
|
14
|
+
* and cleared on scope unwind via `scope.defer(...)`, with a
|
|
15
|
+
* "still mine?" guard so an older op's deferred cleanup cannot stomp
|
|
16
|
+
* a newer op's mapping.
|
|
17
|
+
*
|
|
18
|
+
* A session pre-empts only its own operation: one that finds another
|
|
19
|
+
* session's operation on the workspace is refused with
|
|
20
|
+
* `RAGWorkspaceBusyError`, since a session reaches only its own requests.
|
|
21
|
+
* The owner pre-empts any session's operation.
|
|
22
|
+
*/
|
|
23
|
+
export declare function beginRagContext(workspace: string, request: {
|
|
24
|
+
requestId?: string | undefined;
|
|
25
|
+
sessionId?: string | undefined;
|
|
26
|
+
}): Promise<ManagedRequestContext>;
|
|
2
27
|
export declare function handleRag(request: RagRequest, onProgress?: (update: RagProgressUpdate) => void): Promise<RagResponse>;
|
package/dist/rag/handler.js
CHANGED
|
@@ -53,6 +53,9 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
53
53
|
import { chunk, ingest, reindex, saveEmbeddings, search, deleteEmbeddings, listWorkspaces, closeWorkspace, deleteWorkspace, DEFAULT_WORKSPACE, getActiveRagRequest, setActiveRagRequest, clearActiveRagRequest } from '../rag/index.js';
|
|
54
54
|
import { getRequestRegistry, withRequestContext } from '../runtime/index.js';
|
|
55
55
|
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
56
|
+
import { OWNER_SESSION_ID, resolveSessionId } from '../runtime/session-id.js';
|
|
57
|
+
import { RAGWorkspaceBusyError } from '../errors/index.js';
|
|
58
|
+
import { clearRagEmbedParent, setRagEmbedParent } from '../rag/embedding-scope.js';
|
|
56
59
|
import { getEngineLogger } from '../logging/index.js';
|
|
57
60
|
import { profileReplyHandler, registerOperationMetrics } from '../profiling/index.js';
|
|
58
61
|
registerOperationMetrics({
|
|
@@ -110,20 +113,51 @@ function omitOnProgress(obj) {
|
|
|
110
113
|
* and cleared on scope unwind via `scope.defer(...)`, with a
|
|
111
114
|
* "still mine?" guard so an older op's deferred cleanup cannot stomp
|
|
112
115
|
* a newer op's mapping.
|
|
116
|
+
*
|
|
117
|
+
* A session pre-empts only its own operation: one that finds another
|
|
118
|
+
* session's operation on the workspace is refused with
|
|
119
|
+
* `RAGWorkspaceBusyError`, since a session reaches only its own requests.
|
|
120
|
+
* The owner pre-empts any session's operation.
|
|
113
121
|
*/
|
|
114
|
-
async function beginRagContext(workspace,
|
|
122
|
+
export async function beginRagContext(workspace, request) {
|
|
115
123
|
const registry = getRequestRegistry();
|
|
124
|
+
const requestId = request.requestId ?? generateRandomRequestId();
|
|
125
|
+
const sessionId = resolveSessionId(request);
|
|
126
|
+
const identity = { requestId, sessionId };
|
|
116
127
|
const prev = getActiveRagRequest(workspace);
|
|
117
|
-
|
|
118
|
-
|
|
128
|
+
// The same request sent again: the registry refuses the duplicate id, and
|
|
129
|
+
// the running operation keeps the workspace.
|
|
130
|
+
const duplicate = prev?.requestId === requestId && prev.sessionId === sessionId;
|
|
131
|
+
if (prev !== undefined && !duplicate) {
|
|
132
|
+
if (prev.sessionId !== sessionId && sessionId !== OWNER_SESSION_ID) {
|
|
133
|
+
throw new RAGWorkspaceBusyError(workspace);
|
|
134
|
+
}
|
|
135
|
+
registry.cancel({ ...prev, reason: 'rag-workspace-preempt' });
|
|
119
136
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
137
|
+
// Recorded before the admission wait, so a second call arriving during it
|
|
138
|
+
// sees this operation and is refused or pre-empts it.
|
|
139
|
+
if (!duplicate)
|
|
140
|
+
setActiveRagRequest(workspace, identity);
|
|
141
|
+
let ctx;
|
|
142
|
+
try {
|
|
143
|
+
ctx = await registry.begin({
|
|
144
|
+
requestId,
|
|
145
|
+
sessionId,
|
|
146
|
+
kind: 'rag'
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
if (!duplicate)
|
|
151
|
+
clearActiveRagRequest(workspace, identity);
|
|
152
|
+
throw error;
|
|
153
|
+
}
|
|
154
|
+
// The workspace's embeddings run as children of this request for its
|
|
155
|
+
// duration: the caller's session, aborted with it.
|
|
156
|
+
const embedParent = { signal: ctx.signal, requestId, sessionId: request.sessionId };
|
|
157
|
+
setRagEmbedParent(workspace, embedParent);
|
|
125
158
|
ctx.scope.defer(() => {
|
|
126
|
-
|
|
159
|
+
clearRagEmbedParent(workspace, embedParent);
|
|
160
|
+
clearActiveRagRequest(workspace, identity);
|
|
127
161
|
});
|
|
128
162
|
return ctx;
|
|
129
163
|
}
|
|
@@ -145,8 +179,7 @@ async function handleRagInternal(request, onProgress) {
|
|
|
145
179
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
146
180
|
try {
|
|
147
181
|
const workspace = request.workspace ?? DEFAULT_WORKSPACE;
|
|
148
|
-
const
|
|
149
|
-
const ctx = __addDisposableResource(env_1, await beginRagContext(workspace, requestId), true);
|
|
182
|
+
const ctx = __addDisposableResource(env_1, await beginRagContext(workspace, request), true);
|
|
150
183
|
const log = withRequestContext(getEngineLogger(), ctx);
|
|
151
184
|
log.debug('ingest start');
|
|
152
185
|
const handlerOptions = createHandlerOptions('ingest', workspace, ctx.signal, onProgress);
|
|
@@ -174,8 +207,7 @@ async function handleRagInternal(request, onProgress) {
|
|
|
174
207
|
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
175
208
|
try {
|
|
176
209
|
const workspace = request.workspace ?? DEFAULT_WORKSPACE;
|
|
177
|
-
const
|
|
178
|
-
const ctx = __addDisposableResource(env_2, await beginRagContext(workspace, requestId), true);
|
|
210
|
+
const ctx = __addDisposableResource(env_2, await beginRagContext(workspace, request), true);
|
|
179
211
|
const log = withRequestContext(getEngineLogger(), ctx);
|
|
180
212
|
log.debug('saveEmbeddings start');
|
|
181
213
|
const handlerOptions = createHandlerOptions('saveEmbeddings', workspace, ctx.signal, onProgress);
|
|
@@ -199,7 +231,10 @@ async function handleRagInternal(request, onProgress) {
|
|
|
199
231
|
}
|
|
200
232
|
}
|
|
201
233
|
case 'search': {
|
|
202
|
-
const results = await search(request
|
|
234
|
+
const results = await search(request, {
|
|
235
|
+
requestId: request.requestId,
|
|
236
|
+
sessionId: request.sessionId
|
|
237
|
+
});
|
|
203
238
|
return {
|
|
204
239
|
type: 'rag',
|
|
205
240
|
operation: request.operation,
|
|
@@ -219,8 +254,7 @@ async function handleRagInternal(request, onProgress) {
|
|
|
219
254
|
const env_3 = { stack: [], error: void 0, hasError: false };
|
|
220
255
|
try {
|
|
221
256
|
const workspace = request.workspace ?? DEFAULT_WORKSPACE;
|
|
222
|
-
const
|
|
223
|
-
const ctx = __addDisposableResource(env_3, await beginRagContext(workspace, requestId), true);
|
|
257
|
+
const ctx = __addDisposableResource(env_3, await beginRagContext(workspace, request), true);
|
|
224
258
|
const log = withRequestContext(getEngineLogger(), ctx);
|
|
225
259
|
log.debug('reindex start');
|
|
226
260
|
const handlerOptions = createHandlerOptions('reindex', workspace, ctx.signal, onProgress);
|
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
import { getRagInstance } from '../../rag/rag-workspace-manager.js';
|
|
2
|
-
import {
|
|
2
|
+
import { createWorkspaceEmbeddingFunction } from '../../rag/embedding-scope.js';
|
|
3
3
|
import { ragIngestParamsSchema } from '../../schemas/index.js';
|
|
4
4
|
export async function ingest(params, options) {
|
|
5
5
|
const { modelId, documents, chunk, chunkOpts, workspace, progressInterval } = ragIngestParamsSchema.parse(params);
|
|
6
|
-
|
|
7
|
-
const result = await embed({ modelId, text });
|
|
8
|
-
return result.embedding;
|
|
9
|
-
}
|
|
10
|
-
const rag = await getRagInstance(modelId, embeddingFunction, workspace);
|
|
6
|
+
const rag = await getRagInstance(modelId, createWorkspaceEmbeddingFunction(modelId, workspace), workspace);
|
|
11
7
|
const ingestOpts = {
|
|
12
8
|
chunk,
|
|
13
9
|
chunkOpts,
|