@qvac/inference 0.18.2 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +69 -36
- package/README.md +1 -5
- package/dist/api/assess-model-fit.d.ts +16 -0
- package/dist/api/assess-model-fit.js +27 -0
- package/dist/api/audio-gen.d.ts +4 -4
- package/dist/api/audio-gen.js +17 -6
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/download-asset.d.ts +1 -1
- package/dist/api/download-asset.js +2 -2
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-loaded-model-info.d.ts +6 -10
- package/dist/api/get-loaded-model-info.js +6 -10
- package/dist/api/heartbeat.d.ts +5 -22
- package/dist/api/heartbeat.js +6 -23
- package/dist/api/index.d.ts +36 -35
- package/dist/api/index.js +35 -35
- package/dist/api/load-model.d.ts +5 -1
- package/dist/api/load-model.js +3 -5
- package/dist/api/logging-stream-registry.js +1 -1
- package/dist/api/resume.d.ts +0 -3
- package/dist/api/resume.js +0 -3
- package/dist/api/subscribe-logs.js +1 -1
- package/dist/api/suspend.d.ts +0 -4
- package/dist/api/suspend.js +0 -4
- package/dist/api/translate.d.ts +3 -1
- package/dist/api/translate.js +16 -8
- package/dist/api/unload-model.js +3 -5
- package/dist/api/video-request.d.ts +207 -0
- package/dist/api/video-request.js +18 -0
- package/dist/api/video.d.ts +22 -2
- package/dist/api/video.js +25 -15
- package/dist/api/vla-helpers.d.ts +1 -2
- package/dist/api/vla-helpers.js +1 -2
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.d.ts +71 -0
- package/dist/api/world-result.js +293 -0
- package/dist/api/world.d.ts +143 -0
- package/dist/api/world.js +71 -0
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.js +15 -28
- package/dist/errors/index.d.ts +35 -31
- package/dist/errors/index.js +66 -56
- package/dist/handlers/assess-model-fit.d.ts +10 -0
- package/dist/handlers/assess-model-fit.js +75 -0
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +1 -1
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/download-asset.js +6 -5
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +0 -12
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/get-system-resources.js +8 -0
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +3 -3
- package/dist/handlers/load-model/download-stats.js +5 -5
- package/dist/handlers/load-model/handler.js +6 -8
- package/dist/handlers/load-model/http-verify.d.ts +20 -0
- package/dist/handlers/load-model/http-verify.js +54 -0
- package/dist/handlers/load-model/http.d.ts +12 -2
- package/dist/handlers/load-model/http.js +119 -230
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +1 -1
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +4 -4
- package/dist/handlers/load-model/resolve-session.d.ts +8 -1
- package/dist/handlers/load-model/resolve-session.js +12 -9
- package/dist/handlers/load-model/resolve.d.ts +4 -4
- package/dist/handlers/load-model/resolve.js +18 -16
- package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
- package/dist/handlers/load-model/safe-fetch.js +143 -0
- package/dist/handlers/load-model/types.d.ts +9 -0
- package/dist/handlers/plugin-dispatch.js +1 -4
- package/dist/handlers/plugin-invoke.js +1 -4
- package/dist/handlers/types.d.ts +0 -2
- package/dist/handlers/unload-model.js +1 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +250 -0
- package/dist/models/registry/models.js +2071 -1806
- package/dist/models/registry/resource-profiles.d.ts +18 -0
- package/dist/models/registry/resource-profiles.js +4934 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +12 -12
- package/dist/plugins/builtin/index.js +12 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +79 -17
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.js +1 -1
- package/dist/plugins/ops/load-model.js +1 -1
- package/dist/plugins/ops/transcribe.js +2 -2
- package/dist/plugins/ops/translate.js +4 -12
- package/dist/plugins/ops/unload-model.js +19 -21
- package/dist/plugins/registry.d.ts +1 -0
- package/dist/plugins/registry.js +12 -0
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/envelope.d.ts +0 -1
- package/dist/profiling/envelope.js +0 -10
- package/dist/profiling/events.d.ts +2 -3
- package/dist/profiling/events.js +2 -9
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -14
- package/dist/profiling/index.js +12 -13
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +35 -2
- package/dist/profiling/operation-wrappers.js +12 -15
- package/dist/profiling/profiler.d.ts +1 -2
- package/dist/profiling/profiler.js +7 -26
- package/dist/profiling/types.d.ts +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +2 -6
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.d.ts +4 -2
- package/dist/rag/rag-workspace-manager.js +268 -15
- package/dist/registry.js +8 -58
- package/dist/resources/collector.d.ts +3 -1
- package/dist/resources/collector.js +91 -9
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/model-fit/assess.d.ts +36 -0
- package/dist/resources/model-fit/assess.js +679 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
- package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
- package/dist/resources/model-fit/calibration/fit.js +152 -0
- package/dist/resources/model-fit/calibration/index.d.ts +37 -0
- package/dist/resources/model-fit/calibration/index.js +75 -0
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
- package/dist/resources/model-fit/estimators/llm.d.ts +46 -0
- package/dist/resources/model-fit/estimators/llm.js +222 -0
- package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
- package/dist/resources/model-fit/estimators/whisper.js +77 -0
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/types.d.ts +120 -0
- package/dist/resources/model-fit/types.js +1 -0
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/native.js +15 -0
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +14 -0
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +5 -7
- package/dist/runtime/model-registry.d.ts +0 -14
- package/dist/runtime/model-registry.js +17 -40
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +13 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +4 -5
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +18 -3
- package/dist/schemas/assess-model-fit.d.ts +311 -0
- package/dist/schemas/assess-model-fit.js +182 -0
- package/dist/schemas/audio-gen.d.ts +437 -7
- package/dist/schemas/audio-gen.js +131 -19
- package/dist/schemas/batch-completion-stream.d.ts +5 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.js +73 -27
- package/dist/schemas/bci.d.ts +3 -3
- package/dist/schemas/bci.js +5 -5
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/classification.js +13 -6
- package/dist/schemas/common.d.ts +695 -335
- package/dist/schemas/common.js +33 -31
- package/dist/schemas/completion-event.d.ts +5 -2
- package/dist/schemas/completion-event.js +2 -1
- package/dist/schemas/completion-stream.d.ts +3 -5
- package/dist/schemas/completion-stream.js +4 -8
- package/dist/schemas/config.d.ts +640 -18
- package/dist/schemas/config.js +50 -8
- package/dist/schemas/download-asset.d.ts +19 -1
- package/dist/schemas/download-asset.js +20 -4
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +3 -3
- package/dist/schemas/error.d.ts +2 -2
- package/dist/schemas/error.js +2 -2
- package/dist/schemas/errors.d.ts +2 -6
- package/dist/schemas/errors.js +30 -40
- package/dist/schemas/heartbeat.d.ts +10 -0
- package/dist/schemas/heartbeat.js +8 -0
- package/dist/schemas/index.d.ts +46 -44
- package/dist/schemas/index.js +45 -43
- package/dist/schemas/llamacpp-config.d.ts +32 -10
- package/dist/schemas/llamacpp-config.js +11 -8
- package/dist/schemas/load-model.d.ts +2902 -1857
- package/dist/schemas/load-model.js +52 -52
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +2354 -0
- package/dist/schemas/model-config-schemas.js +46 -0
- package/dist/schemas/model-info.d.ts +10 -20
- package/dist/schemas/model-info.js +4 -25
- package/dist/schemas/model-ops.d.ts +3 -1
- package/dist/schemas/model-ops.js +7 -2
- package/dist/schemas/model-resource-profile.d.ts +109 -0
- package/dist/schemas/model-resource-profile.js +83 -0
- package/dist/schemas/model-src-utils.d.ts +1 -0
- package/dist/schemas/model-src-utils.js +33 -13
- package/dist/schemas/ocr.js +48 -13
- package/dist/schemas/path-validation.js +8 -2
- package/dist/schemas/plugin.d.ts +23 -10
- package/dist/schemas/plugin.js +11 -0
- package/dist/schemas/profiling.d.ts +138 -43
- package/dist/schemas/profiling.js +1 -22
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +0 -2
- package/dist/schemas/reload-config.js +2 -3
- package/dist/schemas/sdcpp-config.d.ts +344 -4
- package/dist/schemas/sdcpp-config.js +504 -17
- package/dist/schemas/system-resources.d.ts +138 -0
- package/dist/schemas/system-resources.js +3 -1
- package/dist/schemas/text-to-speech.js +183 -76
- package/dist/schemas/transcription-config.js +253 -85
- package/dist/schemas/transcription.d.ts +3 -3
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.js +53 -25
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/selection.d.ts +1 -5
- package/dist/selection.js +2 -6
- package/dist/surface.d.ts +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +1 -1
- package/dist/utils/cache/auto-history.js +1 -1
- package/dist/utils/cache/paths.js +5 -5
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/image-dimensions.d.ts +26 -0
- package/dist/utils/image-dimensions.js +46 -0
- package/dist/utils/index.d.ts +13 -10
- package/dist/utils/index.js +13 -10
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-sanitize.d.ts +7 -0
- package/dist/utils/path-sanitize.js +36 -0
- package/dist/utils/path-security.d.ts +0 -7
- package/dist/utils/path-security.js +1 -36
- package/dist/utils/shard-utils.js +3 -3
- package/dist/utils/tool-integration.js +1 -1
- package/dist/utils/tools/dialect.js +1 -1
- package/dist/utils/tools/index.d.ts +2 -2
- package/dist/utils/tools/index.js +2 -2
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +3 -3
- package/dist/utils/url-security.d.ts +17 -0
- package/dist/utils/url-security.js +75 -0
- package/package.json +31 -26
- package/dist/api/provide.d.ts +0 -23
- package/dist/api/provide.js +0 -32
- package/dist/api/stop-provider.d.ts +0 -20
- package/dist/api/stop-provider.js +0 -30
- package/dist/p2p/cancel-delegated.d.ts +0 -3
- package/dist/p2p/cancel-delegated.js +0 -64
- package/dist/p2p/completion-stream-delegated.d.ts +0 -4
- package/dist/p2p/completion-stream-delegated.js +0 -51
- package/dist/p2p/delegate-client.d.ts +0 -9
- package/dist/p2p/delegate-client.js +0 -219
- package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
- package/dist/p2p/delegate-connect-diagnostics.js +0 -38
- package/dist/p2p/delegate-transport.d.ts +0 -18
- package/dist/p2p/delegate-transport.js +0 -237
- package/dist/p2p/error-handlers.d.ts +0 -4
- package/dist/p2p/error-handlers.js +0 -31
- package/dist/p2p/handle-request.d.ts +0 -2
- package/dist/p2p/handle-request.js +0 -171
- package/dist/p2p/handler-utils.d.ts +0 -19
- package/dist/p2p/handler-utils.js +0 -180
- package/dist/p2p/heartbeat-delegated.d.ts +0 -3
- package/dist/p2p/heartbeat-delegated.js +0 -32
- package/dist/p2p/load-model-delegated.d.ts +0 -6
- package/dist/p2p/load-model-delegated.js +0 -132
- package/dist/p2p/progress-throttle.d.ts +0 -7
- package/dist/p2p/progress-throttle.js +0 -47
- package/dist/p2p/provideHandler/connection.d.ts +0 -2
- package/dist/p2p/provideHandler/connection.js +0 -38
- package/dist/p2p/provideHandler/handler.d.ts +0 -2
- package/dist/p2p/provideHandler/handler.js +0 -66
- package/dist/p2p/provideHandler/proxy.d.ts +0 -2
- package/dist/p2p/provideHandler/proxy.js +0 -24
- package/dist/p2p/provideHandler.d.ts +0 -1
- package/dist/p2p/provideHandler.js +0 -2
- package/dist/p2p/rpc-utils.d.ts +0 -4
- package/dist/p2p/rpc-utils.js +0 -8
- package/dist/p2p/stopProvideHandler.d.ts +0 -2
- package/dist/p2p/stopProvideHandler.js +0 -29
- package/dist/p2p/swarm.d.ts +0 -9
- package/dist/p2p/swarm.js +0 -103
- package/dist/p2p/transport-selector.d.ts +0 -2
- package/dist/p2p/transport-selector.js +0 -18
- package/dist/p2p/unload-model-delegated.d.ts +0 -2
- package/dist/p2p/unload-model-delegated.js +0 -31
- package/dist/profiling/delegation-profiler.d.ts +0 -31
- package/dist/profiling/delegation-profiler.js +0 -155
- package/dist/schemas/delegate.d.ts +0 -29
- package/dist/schemas/delegate.js +0 -40
- package/dist/schemas/provide.d.ts +0 -50
- package/dist/schemas/provide.js +0 -32
|
@@ -3,7 +3,7 @@ import path from 'bare-path';
|
|
|
3
3
|
import { unregisterModel } from '../../runtime/model-registry.js';
|
|
4
4
|
import { getRequestRegistry } from '../../runtime/index.js';
|
|
5
5
|
import { unregisterAllLoggingStreams } from '../../runtime/logging-stream-registry.js';
|
|
6
|
-
import { clearFinetuneRuntimeState } from '
|
|
6
|
+
import { clearFinetuneRuntimeState } from '../builtin/llamacpp-completion/ops/finetune.js';
|
|
7
7
|
import { unregisterAddonLogger, getEngineLogger } from '../../logging/index.js';
|
|
8
8
|
import { unloadModelParamsSchema } from '../../schemas/index.js';
|
|
9
9
|
import { ModelNotLoadedError } from '../../errors/index.js';
|
|
@@ -21,27 +21,25 @@ export async function unloadModel(params) {
|
|
|
21
21
|
// lifted for us when this resolves.
|
|
22
22
|
await getRequestRegistry().withModelDraining(modelId, async () => {
|
|
23
23
|
clearFinetuneRuntimeState(modelId);
|
|
24
|
-
if (
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
if (entry.local.model.unload) {
|
|
25
|
+
await entry.local.model.unload();
|
|
26
|
+
}
|
|
27
|
+
if (clearStorage && entry.local.path) {
|
|
28
|
+
const modelPath = entry.local.path;
|
|
29
|
+
const modelFileName = path.basename(modelPath);
|
|
30
|
+
const shardInfo = detectShardedModel(modelFileName);
|
|
31
|
+
if (shardInfo.isSharded) {
|
|
32
|
+
const shardDir = path.dirname(modelPath);
|
|
33
|
+
await fsPromises.rm(shardDir, { recursive: true, force: true });
|
|
34
|
+
logger.info(`Sharded model storage cleared: ${shardDir}`);
|
|
27
35
|
}
|
|
28
|
-
|
|
29
|
-
const
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
logger.info(`Sharded model storage cleared: ${shardDir}`);
|
|
36
|
-
}
|
|
37
|
-
else {
|
|
38
|
-
const target = getClearStorageTarget(modelPath);
|
|
39
|
-
await fsPromises.rm(target.path, {
|
|
40
|
-
recursive: target.kind === 'directory',
|
|
41
|
-
force: true
|
|
42
|
-
});
|
|
43
|
-
logger.info(`Model storage cleared (${target.kind}): ${target.path}`);
|
|
44
|
-
}
|
|
36
|
+
else {
|
|
37
|
+
const target = getClearStorageTarget(modelPath);
|
|
38
|
+
await fsPromises.rm(target.path, {
|
|
39
|
+
recursive: target.kind === 'directory',
|
|
40
|
+
force: true
|
|
41
|
+
});
|
|
42
|
+
logger.info(`Model storage cleared (${target.kind}): ${target.path}`);
|
|
45
43
|
}
|
|
46
44
|
}
|
|
47
45
|
unregisterAddonLogger(modelId);
|
|
@@ -6,4 +6,5 @@ export declare function getPluginHandler(modelType: string, handlerName: string)
|
|
|
6
6
|
export declare function hasPlugin(modelType: string): boolean;
|
|
7
7
|
export declare function unregisterPlugin(modelType: string): boolean;
|
|
8
8
|
export declare function getAllPlugins(): QvacPlugin[];
|
|
9
|
+
export declare function getTurboVecIndexProvider(): import("@qvac/rag").TurboVecIndexProvider | undefined;
|
|
9
10
|
export declare function clearPlugins(): void;
|
package/dist/plugins/registry.js
CHANGED
|
@@ -11,6 +11,9 @@ function getLoggingModule(plugin) {
|
|
|
11
11
|
function findPluginUsingLoggingModule(loggingModule) {
|
|
12
12
|
return Array.from(plugins.values()).find((plugin) => plugin.logging?.module === loggingModule);
|
|
13
13
|
}
|
|
14
|
+
function findPluginProvidingTurboVecIndex() {
|
|
15
|
+
return Array.from(plugins.values()).find((plugin) => plugin.capabilities?.turbovecIndexProvider);
|
|
16
|
+
}
|
|
14
17
|
function getModelTypeForError(plugin) {
|
|
15
18
|
if (!plugin || typeof plugin !== 'object')
|
|
16
19
|
return '(unknown)';
|
|
@@ -33,6 +36,12 @@ export function registerPlugin(plugin) {
|
|
|
33
36
|
if (plugins.has(plugin.modelType)) {
|
|
34
37
|
throw new PluginAlreadyRegisteredError(plugin.modelType);
|
|
35
38
|
}
|
|
39
|
+
const pluginProvidingTurboVecIndex = plugin.capabilities?.turbovecIndexProvider
|
|
40
|
+
? findPluginProvidingTurboVecIndex()
|
|
41
|
+
: undefined;
|
|
42
|
+
if (pluginProvidingTurboVecIndex) {
|
|
43
|
+
throw new PluginDefinitionInvalidError(plugin.modelType, `plugin "${plugin.modelType}" cannot provide turbovecIndexProvider because plugin "${pluginProvidingTurboVecIndex.modelType}" already provides it`);
|
|
44
|
+
}
|
|
36
45
|
// Validate logging module shape if provided
|
|
37
46
|
if (plugin.logging?.module) {
|
|
38
47
|
const loggingModule = plugin.logging.module;
|
|
@@ -84,6 +93,9 @@ export function unregisterPlugin(modelType) {
|
|
|
84
93
|
export function getAllPlugins() {
|
|
85
94
|
return Array.from(plugins.values());
|
|
86
95
|
}
|
|
96
|
+
export function getTurboVecIndexProvider() {
|
|
97
|
+
return findPluginProvidingTurboVecIndex()?.capabilities?.turbovecIndexProvider;
|
|
98
|
+
}
|
|
87
99
|
export function clearPlugins() {
|
|
88
100
|
const loggingModules = new Map();
|
|
89
101
|
for (const plugin of plugins.values()) {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Accumulates profiling statistics and maintains a bounded buffer of recent events.
|
|
3
3
|
* Uses a ring buffer for O(1) event insertion.
|
|
4
4
|
*/
|
|
5
|
-
import type { ProfilingEvent, AggregatedStats } from '
|
|
5
|
+
import type { ProfilingEvent, AggregatedStats } from './types.js';
|
|
6
6
|
export declare function createAggregator(maxRecentEvents?: number): void;
|
|
7
7
|
export declare function recordEvent(event: ProfilingEvent, storeInBuffer?: boolean): void;
|
|
8
8
|
export declare function getAggregates(): Record<string, AggregatedStats>;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Uses a ring buffer for O(1) event insertion.
|
|
4
4
|
*/
|
|
5
5
|
import { getGlobalSingleton } from '../utils/global-singleton.js';
|
|
6
|
-
import { createRingBuffer, ringBufferPush, ringBufferToArray, ringBufferClear, ringBufferResize, ringBufferDroppedCount } from '
|
|
6
|
+
import { createRingBuffer, ringBufferPush, ringBufferToArray, ringBufferClear, ringBufferResize, ringBufferDroppedCount } from './ring-buffer.js';
|
|
7
7
|
const DEFAULT_MAX_RECENT_EVENTS = 1000;
|
|
8
8
|
const AGGREGATOR_STATE_KEY = Symbol.for('@qvac/inference:profiler-aggregator-state');
|
|
9
9
|
function getAggregatorState() {
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* 3. Disabled default
|
|
8
8
|
*/
|
|
9
9
|
import type { ProfilerMode, PerCallProfiling } from '../schemas/index.js';
|
|
10
|
-
import type { ProfilerRuntimeOptions, ProfilingEvent, AggregatedStats } from '
|
|
10
|
+
import type { ProfilerRuntimeOptions, ProfilingEvent, AggregatedStats } from './types.js';
|
|
11
11
|
export interface ResolvedProfilerConfig {
|
|
12
12
|
enabled: boolean;
|
|
13
13
|
mode: ProfilerMode;
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* 3. Disabled default
|
|
8
8
|
*/
|
|
9
9
|
import { getGlobalSingleton } from '../utils/global-singleton.js';
|
|
10
|
-
import { createAggregator, recordEvent as aggregatorRecord, getAggregates as aggregatorGetAggregates, getRecentEvents as aggregatorGetRecentEvents, clearAggregator, setMaxRecentEvents } from '
|
|
11
|
-
import { nowMs, generateProfileId } from '
|
|
10
|
+
import { createAggregator, recordEvent as aggregatorRecord, getAggregates as aggregatorGetAggregates, getRecentEvents as aggregatorGetRecentEvents, clearAggregator, setMaxRecentEvents } from './aggregator.js';
|
|
11
|
+
import { nowMs, generateProfileId } from './clock.js';
|
|
12
12
|
const DEFAULT_CONFIG = {
|
|
13
13
|
enabled: false,
|
|
14
14
|
mode: 'summary',
|
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
import { type ProfilingRequestMeta, type ProfilingResponseMeta } from '../schemas/index.js';
|
|
5
5
|
export declare function createProfilingMeta(profileId: string, includeServerBreakdown: boolean, includeResourceGauges: boolean): ProfilingRequestMeta;
|
|
6
6
|
export declare function createProfilingDisabledMeta(): ProfilingRequestMeta;
|
|
7
|
-
export declare function createDelegatedProfilingMeta(profileId: string, meta?: ProfilingRequestMeta): ProfilingRequestMeta;
|
|
8
7
|
export declare function injectProfilingMetaIntoObject(obj: Record<string, unknown>, meta: ProfilingRequestMeta): Record<string, unknown>;
|
|
9
8
|
export declare function extractProfilingMeta(payload: unknown): ProfilingResponseMeta | undefined;
|
|
10
9
|
export declare function stripProfilingMeta<T extends object>(payload: T): T;
|
|
@@ -13,16 +13,6 @@ export function createProfilingMeta(profileId, includeServerBreakdown, includeRe
|
|
|
13
13
|
export function createProfilingDisabledMeta() {
|
|
14
14
|
return { enabled: false };
|
|
15
15
|
}
|
|
16
|
-
export function createDelegatedProfilingMeta(profileId, meta) {
|
|
17
|
-
return {
|
|
18
|
-
enabled: true,
|
|
19
|
-
id: profileId,
|
|
20
|
-
includeServer: meta?.includeServer ?? false,
|
|
21
|
-
includeResources: meta?.includeResources ?? false,
|
|
22
|
-
resourceOrigin: 'provider',
|
|
23
|
-
...(meta?.mode !== undefined && { mode: meta.mode })
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
16
|
export function injectProfilingMetaIntoObject(obj, meta) {
|
|
27
17
|
return { ...obj, [PROFILING_KEY]: meta };
|
|
28
18
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ProfilingEventKind } from '
|
|
2
|
-
import type { ServerBreakdown
|
|
1
|
+
import type { ProfilingEventKind } from './types.js';
|
|
2
|
+
import type { ServerBreakdown } from '../schemas/index.js';
|
|
3
3
|
export interface BaseTimings {
|
|
4
4
|
profileId: string;
|
|
5
5
|
requestType: string;
|
|
@@ -16,4 +16,3 @@ export declare function recordPhase(base: BaseEvent, phase: string, ms?: number,
|
|
|
16
16
|
}): void;
|
|
17
17
|
export declare function recordFailure(base: BaseEvent, startTime: number, error: unknown): void;
|
|
18
18
|
export declare function recordServerBreakdownPhases(base: BaseEvent, server: ServerBreakdown, prefix?: string): void;
|
|
19
|
-
export declare function recordDelegationBreakdownPhases(base: BaseEvent, delegation: DelegationBreakdown, prefix?: string): void;
|
package/dist/profiling/events.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { nowMs } from '
|
|
2
|
-
import { record } from '
|
|
1
|
+
import { nowMs } from './clock.js';
|
|
2
|
+
import { record } from './controller.js';
|
|
3
3
|
export function recordPhase(base, phase, ms, extra) {
|
|
4
4
|
if (ms === undefined)
|
|
5
5
|
return;
|
|
@@ -26,10 +26,3 @@ export function recordServerBreakdownPhases(base, server, prefix = 'server') {
|
|
|
26
26
|
recordPhase(base, `${prefix}.response.stringify`, server.responseStringifyMs);
|
|
27
27
|
recordPhase(base, `${prefix}.totalServerTime`, server.totalServerMs);
|
|
28
28
|
}
|
|
29
|
-
export function recordDelegationBreakdownPhases(base, delegation, prefix = 'delegation') {
|
|
30
|
-
recordPhase(base, `${prefix}.connection`, delegation.connectionMs);
|
|
31
|
-
recordPhase(base, `${prefix}.request.stringify`, delegation.requestStringifyMs);
|
|
32
|
-
recordPhase(base, `${prefix}.serverWait`, delegation.serverWaitMs);
|
|
33
|
-
recordPhase(base, `${prefix}.response.jsonParse`, delegation.responseJsonParseMs);
|
|
34
|
-
recordPhase(base, `${prefix}.totalDelegationTime`, delegation.totalDelegationMs);
|
|
35
|
-
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* JSON, table, and summary export utilities for profiling data.
|
|
3
3
|
*/
|
|
4
|
-
import type { ProfilerExport } from '
|
|
4
|
+
import type { ProfilerExport } from './types.js';
|
|
5
5
|
export declare function exportJSON(options?: {
|
|
6
6
|
includeRecentEvents?: boolean;
|
|
7
7
|
}): ProfilerExport;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* JSON, table, and summary export utilities for profiling data.
|
|
3
3
|
*/
|
|
4
|
-
import { getEffectiveConfig, getAggregates, getRecentEvents } from '
|
|
5
|
-
import { getDroppedCount, getEventCount } from '
|
|
6
|
-
import { nowMs } from '
|
|
4
|
+
import { getEffectiveConfig, getAggregates, getRecentEvents } from './controller.js';
|
|
5
|
+
import { getDroppedCount, getEventCount } from './aggregator.js';
|
|
6
|
+
import { nowMs } from './clock.js';
|
|
7
7
|
export function exportJSON(options) {
|
|
8
8
|
const config = getEffectiveConfig();
|
|
9
9
|
const includeRecent = options?.includeRecentEvents ?? true;
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* profiler.disable();
|
|
12
12
|
* ```
|
|
13
13
|
*/
|
|
14
|
-
import * as controller from '
|
|
15
|
-
import type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats } from '
|
|
14
|
+
import * as controller from './controller.js';
|
|
15
|
+
import type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats } from './types.js';
|
|
16
16
|
export declare const profiler: {
|
|
17
17
|
/**
|
|
18
18
|
* Enables profiling and resets all previously aggregated data.
|
|
@@ -76,16 +76,15 @@ export declare const profiler: {
|
|
|
76
76
|
*/
|
|
77
77
|
clear: () => void;
|
|
78
78
|
};
|
|
79
|
-
export type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats, ProfilingEventKind } from '
|
|
79
|
+
export type { ProfilerRuntimeOptions, ProfilingEvent, ProfilerExport, AggregatedStats, ProfilingEventKind } from './types.js';
|
|
80
80
|
export type { ProfilerMode, ProfilerGPUResourceGauge, ProfilerResourceGauge } from '../schemas/index.js';
|
|
81
|
-
export { nowMs } from '
|
|
82
|
-
export { attachBackendDiagnostics } from '
|
|
83
|
-
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord, type ResolvedProfilerConfig } from '
|
|
84
|
-
export { recordEvent, clearAggregator } from '
|
|
85
|
-
export { exportJSON, exportTable, exportSummary } from '
|
|
86
|
-
export { createProfilingMeta,
|
|
87
|
-
export { recordPhase, recordFailure, recordServerBreakdownPhases,
|
|
88
|
-
export { createServerProfiler, type ServerProfiler } from '
|
|
89
|
-
export { profileReplyHandler, profileStreamHandler } from '
|
|
90
|
-
export { registerOperationMetrics, buildOperationEvent, type OperationMetricsConfig } from '
|
|
91
|
-
export { shouldProfileDelegation, createDelegationTimings, createDelegationStreamTimings, recordDelegationEvents, recordDelegationStreamEvents, cacheDelegationConnectionTime, flushServerConnectionEvent, consumeBreakdownConnectionTime, clearPeerConnectionTracking, resetDelegationConnectionTracking, type DelegationTimings, type DelegationStreamTimings, type DelegatedHandlerOptions } from '../profiling/delegation-profiler.js';
|
|
81
|
+
export { nowMs } from './clock.js';
|
|
82
|
+
export { attachBackendDiagnostics } from './backend-diagnostics.js';
|
|
83
|
+
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord, type ResolvedProfilerConfig } from './controller.js';
|
|
84
|
+
export { recordEvent, clearAggregator } from './aggregator.js';
|
|
85
|
+
export { exportJSON, exportTable, exportSummary } from './exporters.js';
|
|
86
|
+
export { createProfilingMeta, createProfilingDisabledMeta, injectProfilingMetaIntoObject, extractProfilingMeta, stripProfilingMeta } from './envelope.js';
|
|
87
|
+
export { recordPhase, recordFailure, recordServerBreakdownPhases, type BaseTimings, type BaseEvent } from './events.js';
|
|
88
|
+
export { createServerProfiler, type ServerProfiler } from './profiler.js';
|
|
89
|
+
export { profileReplyHandler, profileStreamHandler } from './operation-wrappers.js';
|
|
90
|
+
export { registerOperationMetrics, buildOperationEvent, type OperationMetricsConfig } from './operation-metrics.js';
|
package/dist/profiling/index.js
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* profiler.disable();
|
|
12
12
|
* ```
|
|
13
13
|
*/
|
|
14
|
-
import * as controller from '
|
|
15
|
-
import * as exporters from '
|
|
14
|
+
import * as controller from './controller.js';
|
|
15
|
+
import * as exporters from './exporters.js';
|
|
16
16
|
export const profiler = {
|
|
17
17
|
/**
|
|
18
18
|
* Enables profiling and resets all previously aggregated data.
|
|
@@ -74,14 +74,13 @@ export const profiler = {
|
|
|
74
74
|
*/
|
|
75
75
|
clear: () => controller.clear()
|
|
76
76
|
};
|
|
77
|
-
export { nowMs } from '
|
|
78
|
-
export { attachBackendDiagnostics } from '
|
|
79
|
-
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord } from '
|
|
80
|
-
export { recordEvent, clearAggregator } from '
|
|
81
|
-
export { exportJSON, exportTable, exportSummary } from '
|
|
82
|
-
export { createProfilingMeta,
|
|
83
|
-
export { recordPhase, recordFailure, recordServerBreakdownPhases
|
|
84
|
-
export { createServerProfiler } from '
|
|
85
|
-
export { profileReplyHandler, profileStreamHandler } from '
|
|
86
|
-
export { registerOperationMetrics, buildOperationEvent } from '
|
|
87
|
-
export { shouldProfileDelegation, createDelegationTimings, createDelegationStreamTimings, recordDelegationEvents, recordDelegationStreamEvents, cacheDelegationConnectionTime, flushServerConnectionEvent, consumeBreakdownConnectionTime, clearPeerConnectionTracking, resetDelegationConnectionTracking } from '../profiling/delegation-profiler.js';
|
|
77
|
+
export { nowMs } from './clock.js';
|
|
78
|
+
export { attachBackendDiagnostics } from './backend-diagnostics.js';
|
|
79
|
+
export { enable, disable, record, shouldProfile, shouldIncludeServerBreakdown, shouldIncludeResourceGauges, generateId, isEnabled, getAggregates, getRecentEvents, getEffectiveConfig, onRecord } from './controller.js';
|
|
80
|
+
export { recordEvent, clearAggregator } from './aggregator.js';
|
|
81
|
+
export { exportJSON, exportTable, exportSummary } from './exporters.js';
|
|
82
|
+
export { createProfilingMeta, createProfilingDisabledMeta, injectProfilingMetaIntoObject, extractProfilingMeta, stripProfilingMeta } from './envelope.js';
|
|
83
|
+
export { recordPhase, recordFailure, recordServerBreakdownPhases } from './events.js';
|
|
84
|
+
export { createServerProfiler } from './profiler.js';
|
|
85
|
+
export { profileReplyHandler, profileStreamHandler } from './operation-wrappers.js';
|
|
86
|
+
export { registerOperationMetrics, buildOperationEvent } from './operation-metrics.js';
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Declarative extraction of operation-level metrics from request/response data.
|
|
3
3
|
* Used by operation wrappers to capture handler-specific profiling events.
|
|
4
4
|
*/
|
|
5
|
-
import type { ProfilingEvent, ProfilingEventKind } from '
|
|
5
|
+
import type { ProfilingEvent, ProfilingEventKind } from './types.js';
|
|
6
6
|
import type { DownloadStats } from '../handlers/load-model/types.js';
|
|
7
7
|
export type MetricExtractor<T> = (data: T) => Record<string, number> | undefined;
|
|
8
8
|
export type TagExtractor<T> = (data: T) => Record<string, string> | undefined;
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Declarative extraction of operation-level metrics from request/response data.
|
|
3
3
|
* Used by operation wrappers to capture handler-specific profiling events.
|
|
4
4
|
*/
|
|
5
|
-
import { readModelExecutionMs } from '
|
|
6
|
-
import { readBackendDiagnostics } from '
|
|
5
|
+
import { readModelExecutionMs } from './model-execution.js';
|
|
6
|
+
import { readBackendDiagnostics } from './backend-diagnostics.js';
|
|
7
7
|
const metricsRegistry = new Map();
|
|
8
8
|
function extractDownloadStatsGauges(stats, gauges) {
|
|
9
9
|
if (stats?.downloadTimeMs !== undefined) {
|
|
@@ -302,6 +302,39 @@ registerOperationMetrics({
|
|
|
302
302
|
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
303
303
|
}
|
|
304
304
|
});
|
|
305
|
+
registerOperationMetrics({
|
|
306
|
+
op: 'worldStepStream',
|
|
307
|
+
kind: 'handler',
|
|
308
|
+
getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
|
|
309
|
+
fromFinalChunk: (res) => {
|
|
310
|
+
if (!res.stats)
|
|
311
|
+
return undefined;
|
|
312
|
+
const gauges = {};
|
|
313
|
+
if (res.stats.modelLoadMs !== undefined)
|
|
314
|
+
gauges['modelLoadMs'] = res.stats.modelLoadMs;
|
|
315
|
+
if (res.stats.stepMs !== undefined)
|
|
316
|
+
gauges['stepMs'] = res.stats.stepMs;
|
|
317
|
+
if (res.stats.totalStepMs !== undefined)
|
|
318
|
+
gauges['totalStepMs'] = res.stats.totalStepMs;
|
|
319
|
+
if (res.stats.totalSteps !== undefined)
|
|
320
|
+
gauges['totalSteps'] = res.stats.totalSteps;
|
|
321
|
+
if (res.stats.totalFrames !== undefined)
|
|
322
|
+
gauges['totalFrames'] = res.stats.totalFrames;
|
|
323
|
+
if (res.stats.frames !== undefined)
|
|
324
|
+
gauges['frames'] = res.stats.frames;
|
|
325
|
+
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
registerOperationMetrics({
|
|
329
|
+
op: 'worldSceneStream',
|
|
330
|
+
kind: 'handler',
|
|
331
|
+
getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
|
|
332
|
+
fromFinalChunk: (res) => {
|
|
333
|
+
if (res.stats?.sceneCreateMs === undefined)
|
|
334
|
+
return undefined;
|
|
335
|
+
return { sceneCreateMs: res.stats.sceneCreateMs };
|
|
336
|
+
}
|
|
337
|
+
});
|
|
305
338
|
registerOperationMetrics({
|
|
306
339
|
op: 'pluginInvoke',
|
|
307
340
|
kind: 'handler',
|
|
@@ -3,11 +3,13 @@
|
|
|
3
3
|
* Used to wrap dispatch/invoke functions with timing capture.
|
|
4
4
|
*/
|
|
5
5
|
import { PROFILING_KEY, OPERATION_EVENT_KEY } from '../schemas/index.js';
|
|
6
|
-
import { nowMs, generateProfileId } from '
|
|
7
|
-
import { record, shouldIncludeResourceGauges, shouldProfile } from '
|
|
6
|
+
import { nowMs, generateProfileId } from './clock.js';
|
|
7
|
+
import { record, shouldIncludeResourceGauges, shouldProfile } from './controller.js';
|
|
8
8
|
import { getResourceCollector } from '../resources/instance.js';
|
|
9
|
-
import { buildOperationEvent } from '
|
|
10
|
-
|
|
9
|
+
import { buildOperationEvent } from './operation-metrics.js';
|
|
10
|
+
function isTerminalChunk(value) {
|
|
11
|
+
return typeof value === 'object' && value !== null && 'done' in value && value.done === true;
|
|
12
|
+
}
|
|
11
13
|
function getRequestProfilingMeta(request) {
|
|
12
14
|
if (!request || typeof request !== 'object') {
|
|
13
15
|
return undefined;
|
|
@@ -20,17 +22,14 @@ function getRequestProfilingMeta(request) {
|
|
|
20
22
|
}
|
|
21
23
|
function resolveOperationProfiling(options) {
|
|
22
24
|
if (options.perCall) {
|
|
23
|
-
return { perCall: options.perCall
|
|
25
|
+
return { perCall: options.perCall };
|
|
24
26
|
}
|
|
25
27
|
const meta = getRequestProfilingMeta(options.request);
|
|
26
28
|
if (!meta) {
|
|
27
|
-
return { perCall: undefined
|
|
29
|
+
return { perCall: undefined };
|
|
28
30
|
}
|
|
29
31
|
if (meta.enabled === false) {
|
|
30
|
-
return {
|
|
31
|
-
perCall: { enabled: false },
|
|
32
|
-
resourceOrigin: meta.resourceOrigin ?? 'local'
|
|
33
|
-
};
|
|
32
|
+
return { perCall: { enabled: false } };
|
|
34
33
|
}
|
|
35
34
|
return {
|
|
36
35
|
perCall: {
|
|
@@ -38,11 +37,10 @@ function resolveOperationProfiling(options) {
|
|
|
38
37
|
includeServerBreakdown: meta.includeServer,
|
|
39
38
|
includeResourceGauges: meta.includeResources,
|
|
40
39
|
mode: meta.mode
|
|
41
|
-
}
|
|
42
|
-
resourceOrigin: meta.resourceOrigin ?? 'local'
|
|
40
|
+
}
|
|
43
41
|
};
|
|
44
42
|
}
|
|
45
|
-
function toProfilerResourceGauge(sample, sampledAt
|
|
43
|
+
function toProfilerResourceGauge(sample, sampledAt) {
|
|
46
44
|
const gpus = sample.gpus.status === 'supported'
|
|
47
45
|
? {
|
|
48
46
|
...sample.gpus,
|
|
@@ -54,7 +52,6 @@ function toProfilerResourceGauge(sample, sampledAt, origin) {
|
|
|
54
52
|
}
|
|
55
53
|
: sample.gpus;
|
|
56
54
|
return {
|
|
57
|
-
origin,
|
|
58
55
|
sampledAt,
|
|
59
56
|
cpu: sample.cpu,
|
|
60
57
|
memory: sample.memory,
|
|
@@ -74,7 +71,7 @@ function buildAndRecordOperationEvent(params) {
|
|
|
74
71
|
if (shouldIncludeResourceGauges(params.profiling.perCall)) {
|
|
75
72
|
const resources = getResourceCollector()?.sample();
|
|
76
73
|
if (resources) {
|
|
77
|
-
event.resources = toProfilerResourceGauge(resources, nowMs()
|
|
74
|
+
event.resources = toProfilerResourceGauge(resources, nowMs());
|
|
78
75
|
}
|
|
79
76
|
}
|
|
80
77
|
record(event);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Response, type ProfilingRequestMeta, type
|
|
1
|
+
import { type Response, type ProfilingRequestMeta, type OperationEvent } from '../schemas/index.js';
|
|
2
2
|
export interface ServerProfilingContext {
|
|
3
3
|
meta: ProfilingRequestMeta;
|
|
4
4
|
requestStart: number;
|
|
@@ -11,7 +11,6 @@ export interface ServerProfilingContext {
|
|
|
11
11
|
export declare function createProfilingContext(meta: ProfilingRequestMeta): ServerProfilingContext;
|
|
12
12
|
export interface ProfilingInjectionOptions {
|
|
13
13
|
ctx?: ServerProfilingContext;
|
|
14
|
-
delegation?: DelegationBreakdown;
|
|
15
14
|
operation?: OperationEvent;
|
|
16
15
|
}
|
|
17
16
|
export declare function injectProfilingIntoString(jsonString: string, options: ProfilingInjectionOptions): string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { responseSchema, PROFILING_KEY,
|
|
2
|
-
import { nowMs } from '
|
|
1
|
+
import { responseSchema, PROFILING_KEY, OPERATION_EVENT_KEY, PROFILING_TRAILER_KEY } from '../schemas/index.js';
|
|
2
|
+
import { nowMs } from './clock.js';
|
|
3
3
|
export function createProfilingContext(meta) {
|
|
4
4
|
return { meta, requestStart: nowMs() };
|
|
5
5
|
}
|
|
@@ -14,22 +14,17 @@ function buildServerBreakdown(ctx) {
|
|
|
14
14
|
};
|
|
15
15
|
}
|
|
16
16
|
export function injectProfilingIntoString(jsonString, options) {
|
|
17
|
-
const { ctx,
|
|
17
|
+
const { ctx, operation } = options;
|
|
18
18
|
const includeServer = ctx?.meta.includeServer ?? false;
|
|
19
|
-
const hasContent = includeServer || !!
|
|
19
|
+
const hasContent = includeServer || !!operation;
|
|
20
20
|
if (!hasContent || !jsonString.endsWith('}')) {
|
|
21
21
|
return jsonString;
|
|
22
22
|
}
|
|
23
|
-
const id = ctx?.meta.id ??
|
|
23
|
+
const id = ctx?.meta.id ?? operation?.profileId ?? '';
|
|
24
24
|
const profilingMeta = { id };
|
|
25
25
|
if (includeServer && ctx) {
|
|
26
26
|
profilingMeta['server'] = buildServerBreakdown(ctx);
|
|
27
27
|
}
|
|
28
|
-
if (delegation) {
|
|
29
|
-
const { profileId: _unused, ...delegationWithoutId } = delegation;
|
|
30
|
-
void _unused;
|
|
31
|
-
profilingMeta['delegation'] = delegationWithoutId;
|
|
32
|
-
}
|
|
33
28
|
if (operation) {
|
|
34
29
|
profilingMeta['operation'] = operation;
|
|
35
30
|
}
|
|
@@ -44,16 +39,10 @@ const noopProfiler = {
|
|
|
44
39
|
if (!response)
|
|
45
40
|
return '';
|
|
46
41
|
const extended = response;
|
|
47
|
-
const delegation = extended[DELEGATION_BREAKDOWN_KEY];
|
|
48
42
|
const operation = extended[OPERATION_EVENT_KEY];
|
|
49
43
|
const json = JSON.stringify(responseSchema.parse(response));
|
|
50
|
-
if (
|
|
51
|
-
|
|
52
|
-
if (delegation)
|
|
53
|
-
opts.delegation = delegation;
|
|
54
|
-
if (operation)
|
|
55
|
-
opts.operation = operation;
|
|
56
|
-
return injectProfilingIntoString(json, opts);
|
|
44
|
+
if (operation) {
|
|
45
|
+
return injectProfilingIntoString(json, { operation });
|
|
57
46
|
}
|
|
58
47
|
return json;
|
|
59
48
|
},
|
|
@@ -64,7 +53,6 @@ function createActiveProfiler(meta) {
|
|
|
64
53
|
const ctx = createProfilingContext(meta);
|
|
65
54
|
let handlerStart = 0;
|
|
66
55
|
let handlerEnded = false;
|
|
67
|
-
let cachedDelegation;
|
|
68
56
|
let cachedOperation;
|
|
69
57
|
return {
|
|
70
58
|
markRequestParsed: (ms) => {
|
|
@@ -86,17 +74,12 @@ function createActiveProfiler(meta) {
|
|
|
86
74
|
serialize: (response, final = true) => {
|
|
87
75
|
if (!response) {
|
|
88
76
|
const opts = { ctx };
|
|
89
|
-
if (cachedDelegation)
|
|
90
|
-
opts.delegation = cachedDelegation;
|
|
91
77
|
if (cachedOperation)
|
|
92
78
|
opts.operation = cachedOperation;
|
|
93
79
|
return injectProfilingIntoString(`{"${PROFILING_TRAILER_KEY}":true}`, opts);
|
|
94
80
|
}
|
|
95
81
|
const extended = response;
|
|
96
|
-
const delegation = extended[DELEGATION_BREAKDOWN_KEY];
|
|
97
82
|
const operation = extended[OPERATION_EVENT_KEY];
|
|
98
|
-
if (delegation)
|
|
99
|
-
cachedDelegation = delegation;
|
|
100
83
|
if (operation)
|
|
101
84
|
cachedOperation = operation;
|
|
102
85
|
const zodStart = nowMs();
|
|
@@ -107,8 +90,6 @@ function createActiveProfiler(meta) {
|
|
|
107
90
|
ctx.responseStringifyMs = (ctx.responseStringifyMs ?? 0) + (nowMs() - stringifyStart);
|
|
108
91
|
if (final) {
|
|
109
92
|
const opts = { ctx };
|
|
110
|
-
if (delegation)
|
|
111
|
-
opts.delegation = delegation;
|
|
112
93
|
if (operation)
|
|
113
94
|
opts.operation = operation;
|
|
114
95
|
return injectProfilingIntoString(json, opts);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ProfilerResourceGauge } from '../schemas/profiling.js';
|
|
2
2
|
import type { InferenceBackendDiagnostics } from '../schemas/system-resources.js';
|
|
3
|
-
export type ProfilingEventKind = 'rpc' | 'handler' | 'download' | 'load'
|
|
3
|
+
export type ProfilingEventKind = 'rpc' | 'handler' | 'download' | 'load';
|
|
4
4
|
export interface ProfilingEvent {
|
|
5
5
|
/** Timestamp when event was recorded (monotonic ms) */
|
|
6
6
|
ts: number;
|
package/dist/rag/handler.js
CHANGED
|
@@ -50,7 +50,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
-
import { chunk, ingest, reindex, saveEmbeddings, search, deleteEmbeddings, listWorkspaces, closeWorkspace, deleteWorkspace, DEFAULT_WORKSPACE, getActiveRagRequest, setActiveRagRequest, clearActiveRagRequest } from '
|
|
53
|
+
import { chunk, ingest, reindex, saveEmbeddings, search, deleteEmbeddings, listWorkspaces, closeWorkspace, deleteWorkspace, DEFAULT_WORKSPACE, getActiveRagRequest, setActiveRagRequest, clearActiveRagRequest } from './index.js';
|
|
54
54
|
import { getRequestRegistry, withRequestContext } from '../runtime/index.js';
|
|
55
55
|
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
56
56
|
import { getEngineLogger } from '../logging/index.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { closeRagInstance, deleteWorkspace, DEFAULT_WORKSPACE } from '
|
|
1
|
+
import { closeRagInstance, deleteWorkspace, DEFAULT_WORKSPACE } from '../rag-workspace-manager.js';
|
|
2
2
|
import { ragCloseWorkspaceParamsSchema } from '../../schemas/index.js';
|
|
3
3
|
export async function closeWorkspace(params) {
|
|
4
4
|
const { workspace, deleteOnClose } = ragCloseWorkspaceParamsSchema.parse(params);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getRagDbAdapter, hasRagWorkspaceStorage } from '
|
|
1
|
+
import { getRagDbAdapter, hasRagWorkspaceStorage } from '../rag-workspace-manager.js';
|
|
2
2
|
import { ragDeleteEmbeddingsParamsSchema } from '../../schemas/index.js';
|
|
3
3
|
import { RAGDeleteFailedError } from '../../errors/index.js';
|
|
4
4
|
export async function deleteEmbeddings(params) {
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { deleteWorkspace as deleteWorkspaceFromManager } from '../rag-workspace-manager.js';
|
|
2
2
|
import { ragDeleteWorkspaceParamsSchema } from '../../schemas/index.js';
|
|
3
|
-
import { RAGWorkspaceNotFoundError
|
|
3
|
+
import { RAGWorkspaceNotFoundError } from '../../errors/index.js';
|
|
4
4
|
export async function deleteWorkspace(params) {
|
|
5
5
|
const { workspace } = ragDeleteWorkspaceParamsSchema.parse(params);
|
|
6
|
-
// Check if workspace is currently in use
|
|
7
|
-
if (isWorkspaceLoaded(workspace)) {
|
|
8
|
-
throw new RAGWorkspaceInUseError(workspace);
|
|
9
|
-
}
|
|
10
6
|
const deleted = await deleteWorkspaceFromManager(workspace);
|
|
11
7
|
if (!deleted) {
|
|
12
8
|
throw new RAGWorkspaceNotFoundError(workspace);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getRagInstance } from '
|
|
1
|
+
import { getRagInstance } from '../rag-workspace-manager.js';
|
|
2
2
|
import { embed } from '../../plugins/ops/embed.js';
|
|
3
3
|
import { ragIngestParamsSchema } from '../../schemas/index.js';
|
|
4
4
|
export async function ingest(params, options) {
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { type RagWorkspaceInfo } from '
|
|
1
|
+
import { type RagWorkspaceInfo } from '../rag-workspace-manager.js';
|
|
2
2
|
export declare function listWorkspaces(): RagWorkspaceInfo[];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getRagDbAdapter, hasRagWorkspaceStorage } from '
|
|
1
|
+
import { getRagDbAdapter, hasRagWorkspaceStorage } from '../rag-workspace-manager.js';
|
|
2
2
|
import { ragReindexParamsSchema } from '../../schemas/index.js';
|
|
3
3
|
import { RAGHyperDBFailedError } from '../../errors/index.js';
|
|
4
4
|
export async function reindex(params, options) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getRagDbAdapter } from '
|
|
1
|
+
import { getRagDbAdapter } from '../rag-workspace-manager.js';
|
|
2
2
|
import { ragSaveEmbeddingsParamsSchema } from '../../schemas/index.js';
|
|
3
3
|
export async function saveEmbeddings(params, options) {
|
|
4
4
|
const { documents, progressInterval, workspace } = ragSaveEmbeddingsParamsSchema.parse(params);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getRagInstance, hasRagWorkspaceStorage } from '
|
|
1
|
+
import { getRagInstance, hasRagWorkspaceStorage } from '../rag-workspace-manager.js';
|
|
2
2
|
import { embed } from '../../plugins/ops/embed.js';
|
|
3
3
|
import { ragSearchParamsSchema } from '../../schemas/index.js';
|
|
4
4
|
export async function search(params) {
|