@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -50,19 +50,20 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import { requestParent } from '../../schemas/plugin.js';
|
|
53
54
|
import { getModelEntry } from '../../runtime/model-registry.js';
|
|
54
55
|
import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LANGUAGES_MAP } from '../../schemas/index.js';
|
|
55
56
|
import { getLangName, detectOne } from '@qvac/langdetect-text';
|
|
56
57
|
import { nowMs } from '../../profiling/index.js';
|
|
57
58
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
58
|
-
import { buildNmtTranslationStats, markNmtCountersStale } from '
|
|
59
|
+
import { buildNmtTranslationStats, markNmtCountersStale } from '../../plugins/ops/translate-stats.js';
|
|
59
60
|
import { ModelNotFoundError, ModelTypeMismatchError, TranslationFailedError } from '../../errors/index.js';
|
|
60
61
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
61
|
-
import { isAddonContextOverflowError } from '
|
|
62
|
+
import { isAddonContextOverflowError } from '../../plugins/builtin/llamacpp-completion/ops/context-overflow.js';
|
|
62
63
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
63
64
|
import { getModelParallel } from '../../utils/config-transform.js';
|
|
64
65
|
import { getEngineLogger } from '../../logging/index.js';
|
|
65
|
-
import { isTranslatePsyModel, buildTranslatePsyMessages, TRANSLATEPSY_GENERATION_PARAMS } from '
|
|
66
|
+
import { isTranslatePsyModel, buildTranslatePsyMessages, TRANSLATEPSY_GENERATION_PARAMS } from '../../plugins/ops/translatepsy-utils.js';
|
|
66
67
|
const AFRICAN_LANGUAGE_ALIASES = new Map([
|
|
67
68
|
['af', 'afr_Latn'],
|
|
68
69
|
['sw', 'swh_Latn'],
|
|
@@ -122,7 +123,7 @@ const LLM_TRANSLATE_GENERATION_PARAMS = {
|
|
|
122
123
|
function shouldSkipPerCallSampling(modelName) {
|
|
123
124
|
return !!modelName && modelName.startsWith('AFRICAN_');
|
|
124
125
|
}
|
|
125
|
-
export async function* translate(params, requestId) {
|
|
126
|
+
export async function* translate(params, requestId, sessionId, invoke) {
|
|
126
127
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
127
128
|
try {
|
|
128
129
|
const { modelId, text, modelType: inputModelType } = params;
|
|
@@ -168,6 +169,8 @@ export async function* translate(params, requestId) {
|
|
|
168
169
|
// fine because the result is dropped either way.
|
|
169
170
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
170
171
|
requestId: requestId ?? generateRandomRequestId(),
|
|
172
|
+
sessionId,
|
|
173
|
+
...requestParent(invoke),
|
|
171
174
|
kind: 'translate',
|
|
172
175
|
modelId,
|
|
173
176
|
// LLM translate joins the completion lane (its cap = the model's own
|
|
@@ -1,2 +1,18 @@
|
|
|
1
1
|
import { type UnloadModelParams } from '../../schemas/index.js';
|
|
2
|
-
export
|
|
2
|
+
export interface UnloadModelResult {
|
|
3
|
+
/** The caller's lease was dropped. False when the caller held none. */
|
|
4
|
+
released: boolean;
|
|
5
|
+
/** The model left memory. False while another session still holds a lease. */
|
|
6
|
+
unloaded: boolean;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Releases the caller session's lease on the model. The model is torn down
|
|
10
|
+
* only when no lease remains; a session that never loaded the model cannot
|
|
11
|
+
* unload it from under the sessions that did.
|
|
12
|
+
*
|
|
13
|
+
* Runs under the model's load lock: a load of the same configuration that
|
|
14
|
+
* arrives during the teardown waits for it and then loads a fresh copy,
|
|
15
|
+
* instead of finding the id unregistered and loading a second native model
|
|
16
|
+
* while the first is still being freed.
|
|
17
|
+
*/
|
|
18
|
+
export declare function unloadModel(params: UnloadModelParams): Promise<UnloadModelResult>;
|
|
@@ -1,17 +1,45 @@
|
|
|
1
1
|
import { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
|
-
import { unregisterModel } from '../../runtime/model-registry.js';
|
|
3
|
+
import { getModelEntry, releaseModelLease, unregisterModel } from '../../runtime/model-registry.js';
|
|
4
4
|
import { getRequestRegistry } from '../../runtime/index.js';
|
|
5
|
+
import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
|
|
5
6
|
import { unregisterAllLoggingStreams } from '../../runtime/logging-stream-registry.js';
|
|
6
|
-
import { clearFinetuneRuntimeState } from '
|
|
7
|
+
import { clearFinetuneRuntimeState } from '../../plugins/builtin/llamacpp-completion/ops/finetune.js';
|
|
7
8
|
import { unregisterAddonLogger, getEngineLogger } from '../../logging/index.js';
|
|
8
9
|
import { unloadModelParamsSchema } from '../../schemas/index.js';
|
|
9
10
|
import { ModelNotLoadedError } from '../../errors/index.js';
|
|
10
11
|
import { detectShardedModel } from '../../utils/index.js';
|
|
11
12
|
import { getClearStorageTarget } from '../../utils/cache/paths.js';
|
|
13
|
+
import { withModelWorkLock } from '../../runtime/model-work-lock.js';
|
|
12
14
|
const logger = getEngineLogger();
|
|
15
|
+
/**
|
|
16
|
+
* Releases the caller session's lease on the model. The model is torn down
|
|
17
|
+
* only when no lease remains; a session that never loaded the model cannot
|
|
18
|
+
* unload it from under the sessions that did.
|
|
19
|
+
*
|
|
20
|
+
* Runs under the model's load lock: a load of the same configuration that
|
|
21
|
+
* arrives during the teardown waits for it and then loads a fresh copy,
|
|
22
|
+
* instead of finding the id unregistered and loading a second native model
|
|
23
|
+
* while the first is still being freed.
|
|
24
|
+
*/
|
|
13
25
|
export async function unloadModel(params) {
|
|
14
|
-
const
|
|
26
|
+
const parsed = unloadModelParamsSchema.parse(params);
|
|
27
|
+
return withModelWorkLock(parsed.modelId, () => unloadModelLocked(parsed));
|
|
28
|
+
}
|
|
29
|
+
async function unloadModelLocked(params) {
|
|
30
|
+
const { modelId, clearStorage, sessionId = OWNER_SESSION_ID } = params;
|
|
31
|
+
if (!getModelEntry(modelId)) {
|
|
32
|
+
throw new ModelNotLoadedError(modelId);
|
|
33
|
+
}
|
|
34
|
+
const remaining = releaseModelLease(modelId, sessionId);
|
|
35
|
+
if (remaining === null) {
|
|
36
|
+
logger.info(`Session ${sessionId} holds no lease on model ${modelId}; nothing released`);
|
|
37
|
+
return { released: false, unloaded: false };
|
|
38
|
+
}
|
|
39
|
+
if (remaining > 0) {
|
|
40
|
+
logger.info(`Model ${modelId} lease released by ${sessionId}; ${remaining} lease(s) remain`);
|
|
41
|
+
return { released: true, unloaded: false };
|
|
42
|
+
}
|
|
15
43
|
const entry = unregisterModel(modelId);
|
|
16
44
|
if (!entry) {
|
|
17
45
|
throw new ModelNotLoadedError(modelId);
|
|
@@ -46,4 +74,5 @@ export async function unloadModel(params) {
|
|
|
46
74
|
unregisterAllLoggingStreams(modelId);
|
|
47
75
|
logger.info(`Model ${modelId} unloaded`);
|
|
48
76
|
});
|
|
77
|
+
return { released: true, unloaded: true };
|
|
49
78
|
}
|
|
@@ -11,7 +11,7 @@ export declare function storageFromBitWidth(bitWidth: unknown): VectorIndexStora
|
|
|
11
11
|
* The vector index feature's only backend: the fabric vector index that the
|
|
12
12
|
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
13
13
|
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
14
|
-
* dimension rule (multiple of 8, at most
|
|
14
|
+
* dimension rule (multiple of 8, at most 16384), which the native index
|
|
15
15
|
* enforces itself.
|
|
16
16
|
*/
|
|
17
17
|
export declare function getTurboVecBackend(): VectorIndexBackend | undefined;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getTurboVecIndexProvider } from '
|
|
1
|
+
import { getTurboVecIndexProvider } from '../plugins/registry.js';
|
|
2
2
|
import { VECTOR_ID_RESERVED, VectorIndexStorage } from '../schemas/index.js';
|
|
3
3
|
// The native index pads short result rows with UINT64_MAX.
|
|
4
4
|
const PADDING_ID = BigInt(VECTOR_ID_RESERVED);
|
|
@@ -48,6 +48,21 @@ function openIndex(index, storage) {
|
|
|
48
48
|
return index.length;
|
|
49
49
|
},
|
|
50
50
|
storage,
|
|
51
|
+
get calibrated() {
|
|
52
|
+
return index.calibrated;
|
|
53
|
+
},
|
|
54
|
+
get stagedSearch() {
|
|
55
|
+
return index.stagedSearch;
|
|
56
|
+
},
|
|
57
|
+
setStagedSearch(enabled) {
|
|
58
|
+
// Without the getter the setting could not be confirmed, so a provider
|
|
59
|
+
// needs both halves of the control.
|
|
60
|
+
if (!index.setStagedSearch || typeof index.stagedSearch !== 'boolean') {
|
|
61
|
+
throw new Error('The vector index provider does not support staged search controls');
|
|
62
|
+
}
|
|
63
|
+
index.setStagedSearch(enabled);
|
|
64
|
+
return index.stagedSearch;
|
|
65
|
+
},
|
|
51
66
|
add(vectors, ids) {
|
|
52
67
|
index.addWithIds(vectors, ids);
|
|
53
68
|
needsPrepare = true;
|
|
@@ -80,7 +95,7 @@ function openIndex(index, storage) {
|
|
|
80
95
|
* The vector index feature's only backend: the fabric vector index that the
|
|
81
96
|
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
82
97
|
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
83
|
-
* dimension rule (multiple of 8, at most
|
|
98
|
+
* dimension rule (multiple of 8, at most 16384), which the native index
|
|
84
99
|
* enforces itself.
|
|
85
100
|
*/
|
|
86
101
|
export function getTurboVecBackend() {
|
|
@@ -88,11 +103,11 @@ export function getTurboVecBackend() {
|
|
|
88
103
|
if (!provider)
|
|
89
104
|
return undefined;
|
|
90
105
|
return {
|
|
91
|
-
create(dim, storage) {
|
|
92
|
-
return openIndex(provider.create({ dim, storage }), storage);
|
|
106
|
+
async create(dim, storage) {
|
|
107
|
+
return openIndex(await provider.create({ dim, storage }), storage);
|
|
93
108
|
},
|
|
94
|
-
load(path) {
|
|
95
|
-
const index = provider.load(path);
|
|
109
|
+
async load(path) {
|
|
110
|
+
const index = await provider.load(path);
|
|
96
111
|
return openIndex(index, storageFromBitWidth(index.bitWidth));
|
|
97
112
|
}
|
|
98
113
|
};
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Accumulates profiling statistics and maintains a bounded buffer of recent events.
|
|
3
3
|
* Uses a ring buffer for O(1) event insertion.
|
|
4
4
|
*/
|
|
5
|
-
import type { ProfilingEvent, AggregatedStats } from '
|
|
5
|
+
import type { ProfilingEvent, AggregatedStats } from '../profiling/types.js';
|
|
6
6
|
export declare function createAggregator(maxRecentEvents?: number): void;
|
|
7
7
|
export declare function recordEvent(event: ProfilingEvent, storeInBuffer?: boolean): void;
|
|
8
8
|
export declare function getAggregates(): Record<string, AggregatedStats>;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Uses a ring buffer for O(1) event insertion.
|
|
4
4
|
*/
|
|
5
5
|
import { getGlobalSingleton } from '../utils/global-singleton.js';
|
|
6
|
-
import { createRingBuffer, ringBufferPush, ringBufferToArray, ringBufferClear, ringBufferResize, ringBufferDroppedCount } from '
|
|
6
|
+
import { createRingBuffer, ringBufferPush, ringBufferToArray, ringBufferClear, ringBufferResize, ringBufferDroppedCount } from '../profiling/ring-buffer.js';
|
|
7
7
|
const DEFAULT_MAX_RECENT_EVENTS = 1000;
|
|
8
8
|
const AGGREGATOR_STATE_KEY = Symbol.for('@qvac/inference:profiler-aggregator-state');
|
|
9
9
|
function getAggregatorState() {
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* 3. Disabled default
|
|
8
8
|
*/
|
|
9
9
|
import type { ProfilerMode, PerCallProfiling } from '../schemas/index.js';
|
|
10
|
-
import type { ProfilerRuntimeOptions, ProfilingEvent, AggregatedStats } from '
|
|
10
|
+
import type { ProfilerRuntimeOptions, ProfilingEvent, AggregatedStats } from '../profiling/types.js';
|
|
11
11
|
export interface ResolvedProfilerConfig {
|
|
12
12
|
enabled: boolean;
|
|
13
13
|
mode: ProfilerMode;
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* 3. Disabled default
|
|
8
8
|
*/
|
|
9
9
|
import { getGlobalSingleton } from '../utils/global-singleton.js';
|
|
10
|
-
import { createAggregator, recordEvent as aggregatorRecord, getAggregates as aggregatorGetAggregates, getRecentEvents as aggregatorGetRecentEvents, clearAggregator, setMaxRecentEvents } from '
|
|
11
|
-
import { nowMs, generateProfileId } from '
|
|
10
|
+
import { createAggregator, recordEvent as aggregatorRecord, getAggregates as aggregatorGetAggregates, getRecentEvents as aggregatorGetRecentEvents, clearAggregator, setMaxRecentEvents } from '../profiling/aggregator.js';
|
|
11
|
+
import { nowMs, generateProfileId } from '../profiling/clock.js';
|
|
12
12
|
const DEFAULT_CONFIG = {
|
|
13
13
|
enabled: false,
|
|
14
14
|
mode: 'summary',
|
package/dist/profiling/events.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { nowMs } from '
|
|
2
|
-
import { record } from '
|
|
1
|
+
import { nowMs } from '../profiling/clock.js';
|
|
2
|
+
import { record } from '../profiling/controller.js';
|
|
3
3
|
export function recordPhase(base, phase, ms, extra) {
|
|
4
4
|
if (ms === undefined)
|
|
5
5
|
return;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* JSON, table, and summary export utilities for profiling data.
|
|
3
3
|
*/
|
|
4
|
-
import type { ProfilerExport } from '
|
|
4
|
+
import type { ProfilerExport } from '../profiling/types.js';
|
|
5
5
|
export declare function exportJSON(options?: {
|
|
6
6
|
includeRecentEvents?: boolean;
|
|
7
7
|
}): ProfilerExport;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* JSON, table, and summary export utilities for profiling data.
|
|
3
3
|
*/
|
|
4
|
-
import { getEffectiveConfig, getAggregates, getRecentEvents } from '
|
|
5
|
-
import { getDroppedCount, getEventCount } from '
|
|
6
|
-
import { nowMs } from '
|
|
4
|
+
import { getEffectiveConfig, getAggregates, getRecentEvents } from '../profiling/controller.js';
|
|
5
|
+
import { getDroppedCount, getEventCount } from '../profiling/aggregator.js';
|
|
6
|
+
import { nowMs } from '../profiling/clock.js';
|
|
7
7
|
export function exportJSON(options) {
|
|
8
8
|
const config = getEffectiveConfig();
|
|
9
9
|
const includeRecent = options?.includeRecentEvents ?? true;
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* profiler.disable();
|
|
12
12
|
* ```
|
|
13
13
|
*/
|
|
14
|
-
import * as controller from '
|
|
15
|
-
import type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats } from '
|
|
14
|
+
import * as controller from '../profiling/controller.js';
|
|
15
|
+
import type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats } from '../profiling/types.js';
|
|
16
16
|
export declare const profiler: {
|
|
17
17
|
/**
|
|
18
18
|
* Enables profiling and resets all previously aggregated data.
|
|
@@ -76,15 +76,15 @@ export declare const profiler: {
|
|
|
76
76
|
*/
|
|
77
77
|
clear: () => void;
|
|
78
78
|
};
|
|
79
|
-
export type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats, ProfilingEventKind } from '
|
|
79
|
+
export type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats, ProfilingEventKind } from '../profiling/types.js';
|
|
80
80
|
export type { ProfilerMode, ProfilerGPUResourceGauge, ProfilerResourceGauge } from '../schemas/index.js';
|
|
81
|
-
export { nowMs } from '
|
|
82
|
-
export { attachBackendDiagnostics } from '
|
|
83
|
-
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord, type ResolvedProfilerConfig } from '
|
|
84
|
-
export { recordEvent, clearAggregator } from '
|
|
85
|
-
export { exportJSON, exportTable, exportSummary } from '
|
|
86
|
-
export { createProfilingMeta, createProfilingDisabledMeta, injectProfilingMetaIntoObject, extractProfilingMeta, stripProfilingMeta } from '
|
|
87
|
-
export { recordPhase, recordFailure, recordServerBreakdownPhases, type BaseTimings, type BaseEvent } from '
|
|
88
|
-
export { createServerProfiler, type ServerProfiler } from '
|
|
89
|
-
export { profileReplyHandler, profileStreamHandler } from '
|
|
90
|
-
export { registerOperationMetrics, buildOperationEvent, type OperationMetricsConfig } from '
|
|
81
|
+
export { nowMs } from '../profiling/clock.js';
|
|
82
|
+
export { attachBackendDiagnostics } from '../profiling/backend-diagnostics.js';
|
|
83
|
+
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord, type ResolvedProfilerConfig } from '../profiling/controller.js';
|
|
84
|
+
export { recordEvent, clearAggregator } from '../profiling/aggregator.js';
|
|
85
|
+
export { exportJSON, exportTable, exportSummary } from '../profiling/exporters.js';
|
|
86
|
+
export { createProfilingMeta, createProfilingDisabledMeta, injectProfilingMetaIntoObject, extractProfilingMeta, stripProfilingMeta } from '../profiling/envelope.js';
|
|
87
|
+
export { recordPhase, recordFailure, recordServerBreakdownPhases, type BaseTimings, type BaseEvent } from '../profiling/events.js';
|
|
88
|
+
export { createServerProfiler, type ServerProfiler } from '../profiling/profiler.js';
|
|
89
|
+
export { profileReplyHandler, profileStreamHandler } from '../profiling/operation-wrappers.js';
|
|
90
|
+
export { registerOperationMetrics, buildOperationEvent, type OperationMetricsConfig } from '../profiling/operation-metrics.js';
|
package/dist/profiling/index.js
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* profiler.disable();
|
|
12
12
|
* ```
|
|
13
13
|
*/
|
|
14
|
-
import * as controller from '
|
|
15
|
-
import * as exporters from '
|
|
14
|
+
import * as controller from '../profiling/controller.js';
|
|
15
|
+
import * as exporters from '../profiling/exporters.js';
|
|
16
16
|
export const profiler = {
|
|
17
17
|
/**
|
|
18
18
|
* Enables profiling and resets all previously aggregated data.
|
|
@@ -74,13 +74,13 @@ export const profiler = {
|
|
|
74
74
|
*/
|
|
75
75
|
clear: () => controller.clear()
|
|
76
76
|
};
|
|
77
|
-
export { nowMs } from '
|
|
78
|
-
export { attachBackendDiagnostics } from '
|
|
79
|
-
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord } from '
|
|
80
|
-
export { recordEvent, clearAggregator } from '
|
|
81
|
-
export { exportJSON, exportTable, exportSummary } from '
|
|
82
|
-
export { createProfilingMeta, createProfilingDisabledMeta, injectProfilingMetaIntoObject, extractProfilingMeta, stripProfilingMeta } from '
|
|
83
|
-
export { recordPhase, recordFailure, recordServerBreakdownPhases } from '
|
|
84
|
-
export { createServerProfiler } from '
|
|
85
|
-
export { profileReplyHandler, profileStreamHandler } from '
|
|
86
|
-
export { registerOperationMetrics, buildOperationEvent } from '
|
|
77
|
+
export { nowMs } from '../profiling/clock.js';
|
|
78
|
+
export { attachBackendDiagnostics } from '../profiling/backend-diagnostics.js';
|
|
79
|
+
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord } from '../profiling/controller.js';
|
|
80
|
+
export { recordEvent, clearAggregator } from '../profiling/aggregator.js';
|
|
81
|
+
export { exportJSON, exportTable, exportSummary } from '../profiling/exporters.js';
|
|
82
|
+
export { createProfilingMeta, createProfilingDisabledMeta, injectProfilingMetaIntoObject, extractProfilingMeta, stripProfilingMeta } from '../profiling/envelope.js';
|
|
83
|
+
export { recordPhase, recordFailure, recordServerBreakdownPhases } from '../profiling/events.js';
|
|
84
|
+
export { createServerProfiler } from '../profiling/profiler.js';
|
|
85
|
+
export { profileReplyHandler, profileStreamHandler } from '../profiling/operation-wrappers.js';
|
|
86
|
+
export { registerOperationMetrics, buildOperationEvent } from '../profiling/operation-metrics.js';
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Declarative extraction of operation-level metrics from request/response data.
|
|
3
3
|
* Used by operation wrappers to capture handler-specific profiling events.
|
|
4
4
|
*/
|
|
5
|
-
import type { ProfilingEvent, ProfilingEventKind } from '
|
|
5
|
+
import type { ProfilingEvent, ProfilingEventKind } from '../profiling/types.js';
|
|
6
6
|
import type { DownloadStats } from '../handlers/load-model/types.js';
|
|
7
7
|
export type MetricExtractor<T> = (data: T) => Record<string, number> | undefined;
|
|
8
8
|
export type TagExtractor<T> = (data: T) => Record<string, string> | undefined;
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Declarative extraction of operation-level metrics from request/response data.
|
|
3
3
|
* Used by operation wrappers to capture handler-specific profiling events.
|
|
4
4
|
*/
|
|
5
|
-
import { readModelExecutionMs } from '
|
|
6
|
-
import { readBackendDiagnostics } from '
|
|
5
|
+
import { readModelExecutionMs } from '../profiling/model-execution.js';
|
|
6
|
+
import { readBackendDiagnostics } from '../profiling/backend-diagnostics.js';
|
|
7
7
|
const metricsRegistry = new Map();
|
|
8
8
|
function extractDownloadStatsGauges(stats, gauges) {
|
|
9
9
|
if (stats?.downloadTimeMs !== undefined) {
|
|
@@ -54,7 +54,8 @@ export function buildOperationEvent(op, profileId, ts, executionMs, request, fin
|
|
|
54
54
|
Object.assign(gauges, extracted);
|
|
55
55
|
}
|
|
56
56
|
const requestTags = config.getTags?.(request);
|
|
57
|
-
|
|
57
|
+
// A handler that threw has no response; its event carries request tags only.
|
|
58
|
+
const resultTags = finalResponse ? config.getTagsFromResult?.(finalResponse) : undefined;
|
|
58
59
|
const tags = { ...requestTags, ...resultTags };
|
|
59
60
|
const hasGauges = Object.keys(gauges).length > 0;
|
|
60
61
|
const hasTags = Object.keys(tags).length > 0;
|
|
@@ -257,25 +258,27 @@ registerOperationMetrics({
|
|
|
257
258
|
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
258
259
|
}
|
|
259
260
|
});
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
261
|
+
for (const op of ['embed', 'embedBatch']) {
|
|
262
|
+
registerOperationMetrics({
|
|
263
|
+
op,
|
|
264
|
+
kind: 'handler',
|
|
265
|
+
getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
|
|
266
|
+
fromResult: (res) => {
|
|
267
|
+
const gauges = {};
|
|
268
|
+
const modelExecMs = readModelExecutionMs(res);
|
|
269
|
+
if (modelExecMs !== undefined)
|
|
270
|
+
gauges['modelExecutionTime'] = modelExecMs;
|
|
271
|
+
if (res.stats?.totalTime !== undefined)
|
|
272
|
+
gauges['totalTime'] = res.stats.totalTime;
|
|
273
|
+
if (res.stats?.tokensPerSecond !== undefined) {
|
|
274
|
+
gauges['tokensPerSecond'] = res.stats.tokensPerSecond;
|
|
275
|
+
}
|
|
276
|
+
if (res.stats?.totalTokens !== undefined)
|
|
277
|
+
gauges['totalTokens'] = res.stats.totalTokens;
|
|
278
|
+
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
273
279
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
277
|
-
}
|
|
278
|
-
});
|
|
280
|
+
});
|
|
281
|
+
}
|
|
279
282
|
registerOperationMetrics({
|
|
280
283
|
op: 'ocrStream',
|
|
281
284
|
kind: 'handler',
|
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
* Used to wrap dispatch/invoke functions with timing capture.
|
|
4
4
|
*/
|
|
5
5
|
import { PROFILING_KEY, OPERATION_EVENT_KEY } from '../schemas/index.js';
|
|
6
|
-
import { nowMs, generateProfileId } from '
|
|
7
|
-
import { record, shouldIncludeResourceGauges, shouldProfile } from '
|
|
6
|
+
import { nowMs, generateProfileId } from '../profiling/clock.js';
|
|
7
|
+
import { record, shouldIncludeResourceGauges, shouldProfile } from '../profiling/controller.js';
|
|
8
8
|
import { getResourceCollector } from '../resources/instance.js';
|
|
9
|
-
import { buildOperationEvent } from '
|
|
9
|
+
import { buildOperationEvent } from '../profiling/operation-metrics.js';
|
|
10
10
|
function isTerminalChunk(value) {
|
|
11
11
|
return typeof value === 'object' && value !== null && 'done' in value && value.done === true;
|
|
12
12
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { responseSchema, PROFILING_KEY, OPERATION_EVENT_KEY, PROFILING_TRAILER_KEY } from '../schemas/index.js';
|
|
2
|
-
import { nowMs } from '
|
|
2
|
+
import { nowMs } from '../profiling/clock.js';
|
|
3
3
|
export function createProfilingContext(meta) {
|
|
4
4
|
return { meta, requestStart: nowMs() };
|
|
5
5
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type EmbeddingFunction } from '@qvac/rag';
|
|
2
|
+
import type { PluginHandlerContext } from '../schemas/plugin.js';
|
|
3
|
+
export declare function setRagEmbedParent(workspace: string | undefined, parent: PluginHandlerContext): void;
|
|
4
|
+
/** Clears the parent only while it is still the one set, so an older operation's unwind never drops a newer one. */
|
|
5
|
+
export declare function clearRagEmbedParent(workspace: string | undefined, parent: PluginHandlerContext): void;
|
|
6
|
+
export declare function getRagEmbedParent(workspace: string | undefined): PluginHandlerContext | undefined;
|
|
7
|
+
/**
|
|
8
|
+
* The text the workspace embedding function is asked for when a search runs:
|
|
9
|
+
* `@qvac/rag` trims a query before it embeds it, so a search embeds and
|
|
10
|
+
* stashes the trimmed text to be found by the same key.
|
|
11
|
+
*/
|
|
12
|
+
export declare function queryEmbeddingText(query: string): string;
|
|
13
|
+
export declare function stashQueryEmbedding(workspace: string | undefined, query: string, vector: number[]): void;
|
|
14
|
+
export declare function takeQueryEmbedding(workspace: string | undefined, query: string): number[] | undefined;
|
|
15
|
+
/** The embedding function a workspace's RAG instance is created with. */
|
|
16
|
+
export declare function createWorkspaceEmbeddingFunction(modelId: string, workspace: string | undefined): EmbeddingFunction;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { ERR_CODES, QvacErrorRAG } from '@qvac/rag';
|
|
2
|
+
import { InferenceCancelledError } from '../errors/index.js';
|
|
3
|
+
import { embed } from '../plugins/ops/embed.js';
|
|
4
|
+
import { getWorkspaceKey } from '../rag/rag-operation-manager.js';
|
|
5
|
+
/**
|
|
6
|
+
* The request a workspace's embeddings run under.
|
|
7
|
+
*
|
|
8
|
+
* A workspace holds one RAG instance with one embedding function for its
|
|
9
|
+
* lifetime, so the function cannot be bound to the request that is using it.
|
|
10
|
+
* Instead, the registered RAG operation on the workspace (`ingest`,
|
|
11
|
+
* `saveEmbeddings`, `reindex`; one at a time, see `beginRagContext`) publishes
|
|
12
|
+
* itself here for its duration, and the embedding function begins every
|
|
13
|
+
* `embed` as a child of it: the embedding runs as the caller's session and
|
|
14
|
+
* aborts with the operation. A `search` is not a registered operation and
|
|
15
|
+
* runs beside them, so it embeds its query itself, under its own request, and
|
|
16
|
+
* leaves the vector here for the function to hand back.
|
|
17
|
+
*/
|
|
18
|
+
const parentByWorkspace = new Map();
|
|
19
|
+
const queryVectorsByWorkspace = new Map();
|
|
20
|
+
export function setRagEmbedParent(workspace, parent) {
|
|
21
|
+
parentByWorkspace.set(getWorkspaceKey(workspace), parent);
|
|
22
|
+
}
|
|
23
|
+
/** Clears the parent only while it is still the one set, so an older operation's unwind never drops a newer one. */
|
|
24
|
+
export function clearRagEmbedParent(workspace, parent) {
|
|
25
|
+
const key = getWorkspaceKey(workspace);
|
|
26
|
+
if (parentByWorkspace.get(key) === parent)
|
|
27
|
+
parentByWorkspace.delete(key);
|
|
28
|
+
}
|
|
29
|
+
export function getRagEmbedParent(workspace) {
|
|
30
|
+
return parentByWorkspace.get(getWorkspaceKey(workspace));
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* The text the workspace embedding function is asked for when a search runs:
|
|
34
|
+
* `@qvac/rag` trims a query before it embeds it, so a search embeds and
|
|
35
|
+
* stashes the trimmed text to be found by the same key.
|
|
36
|
+
*/
|
|
37
|
+
export function queryEmbeddingText(query) {
|
|
38
|
+
return query.trim();
|
|
39
|
+
}
|
|
40
|
+
export function stashQueryEmbedding(workspace, query, vector) {
|
|
41
|
+
const key = getWorkspaceKey(workspace);
|
|
42
|
+
let vectors = queryVectorsByWorkspace.get(key);
|
|
43
|
+
if (!vectors) {
|
|
44
|
+
vectors = new Map();
|
|
45
|
+
queryVectorsByWorkspace.set(key, vectors);
|
|
46
|
+
}
|
|
47
|
+
vectors.set(query, vector);
|
|
48
|
+
}
|
|
49
|
+
export function takeQueryEmbedding(workspace, query) {
|
|
50
|
+
const key = getWorkspaceKey(workspace);
|
|
51
|
+
const vectors = queryVectorsByWorkspace.get(key);
|
|
52
|
+
const vector = vectors?.get(query);
|
|
53
|
+
if (vectors && vector !== undefined) {
|
|
54
|
+
vectors.delete(query);
|
|
55
|
+
if (vectors.size === 0)
|
|
56
|
+
queryVectorsByWorkspace.delete(key);
|
|
57
|
+
}
|
|
58
|
+
return vector;
|
|
59
|
+
}
|
|
60
|
+
/** The embedding function a workspace's RAG instance is created with. */
|
|
61
|
+
export function createWorkspaceEmbeddingFunction(modelId, workspace) {
|
|
62
|
+
return async (text) => {
|
|
63
|
+
if (typeof text === 'string') {
|
|
64
|
+
const stashed = takeQueryEmbedding(workspace, text);
|
|
65
|
+
if (stashed !== undefined)
|
|
66
|
+
return stashed;
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
const result = await embed({ modelId, text }, undefined, undefined, getRagEmbedParent(workspace));
|
|
70
|
+
return result.embedding;
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
// `@qvac/rag` reports any other error from this function as a failed
|
|
74
|
+
// generation, so a cancelled embed is handed back as its cancellation.
|
|
75
|
+
if (error instanceof InferenceCancelledError) {
|
|
76
|
+
throw new QvacErrorRAG({ code: ERR_CODES.OPERATION_CANCELLED, cause: error });
|
|
77
|
+
}
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
}
|
package/dist/rag/handler.d.ts
CHANGED
|
@@ -1,2 +1,27 @@
|
|
|
1
1
|
import type { RagRequest, RagResponse, RagProgressUpdate } from '../schemas/index.js';
|
|
2
|
+
import { type ManagedRequestContext } from '../runtime/index.js';
|
|
3
|
+
/**
|
|
4
|
+
* Begin a registry-tracked RAG context with workspace-level pre-emption.
|
|
5
|
+
*
|
|
6
|
+
* Workspace-level admission lives in the dispatcher rather than as a
|
|
7
|
+
* registry policy primitive (it's a dispatch concern, not a registry
|
|
8
|
+
* `kind` admission rule). The sequence is **cancel-prior → begin-new**:
|
|
9
|
+
* if another RAG operation is already running on the same workspace,
|
|
10
|
+
* cancel it first, then begin the new context. Reversing the order
|
|
11
|
+
* would cancel the just-installed context.
|
|
12
|
+
*
|
|
13
|
+
* The workspace → requestId map is updated after `begin(...)` succeeds
|
|
14
|
+
* and cleared on scope unwind via `scope.defer(...)`, with a
|
|
15
|
+
* "still mine?" guard so an older op's deferred cleanup cannot stomp
|
|
16
|
+
* a newer op's mapping.
|
|
17
|
+
*
|
|
18
|
+
* A session pre-empts only its own operation: one that finds another
|
|
19
|
+
* session's operation on the workspace is refused with
|
|
20
|
+
* `RAGWorkspaceBusyError`, since a session reaches only its own requests.
|
|
21
|
+
* The owner pre-empts any session's operation.
|
|
22
|
+
*/
|
|
23
|
+
export declare function beginRagContext(workspace: string, request: {
|
|
24
|
+
requestId?: string | undefined;
|
|
25
|
+
sessionId?: string | undefined;
|
|
26
|
+
}): Promise<ManagedRequestContext>;
|
|
2
27
|
export declare function handleRag(request: RagRequest, onProgress?: (update: RagProgressUpdate) => void): Promise<RagResponse>;
|