@qvac/inference 0.0.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +179 -0
- package/NOTICE +986 -0
- package/README.md +138 -0
- package/dist/api/audio-gen.d.ts +18 -0
- package/dist/api/audio-gen.js +148 -0
- package/dist/api/batch-completion.d.ts +14 -0
- package/dist/api/batch-completion.js +332 -0
- package/dist/api/bci-transcribe.d.ts +56 -0
- package/dist/api/bci-transcribe.js +172 -0
- package/dist/api/cancel.d.ts +34 -0
- package/dist/api/cancel.js +82 -0
- package/dist/api/classify.d.ts +22 -0
- package/dist/api/classify.js +46 -0
- package/dist/api/completion-stream.d.ts +106 -0
- package/dist/api/completion-stream.js +351 -0
- package/dist/api/delete-cache.d.ts +29 -0
- package/dist/api/delete-cache.js +49 -0
- package/dist/api/diffusion.d.ts +92 -0
- package/dist/api/diffusion.js +176 -0
- package/dist/api/download-asset.d.ts +49 -0
- package/dist/api/download-asset.js +87 -0
- package/dist/api/embed.d.ts +41 -0
- package/dist/api/embed.js +29 -0
- package/dist/api/finetune.d.ts +89 -0
- package/dist/api/finetune.js +138 -0
- package/dist/api/get-loaded-model-info.d.ts +24 -0
- package/dist/api/get-loaded-model-info.js +34 -0
- package/dist/api/get-model-info.d.ts +43 -0
- package/dist/api/get-model-info.js +21 -0
- package/dist/api/get-system-resources.d.ts +2 -0
- package/dist/api/get-system-resources.js +16 -0
- package/dist/api/heartbeat.d.ts +29 -0
- package/dist/api/heartbeat.js +37 -0
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +35 -0
- package/dist/api/invoke-plugin.d.ts +30 -0
- package/dist/api/invoke-plugin.js +54 -0
- package/dist/api/load-model.d.ts +169 -0
- package/dist/api/load-model.js +129 -0
- package/dist/api/logging-stream-registry.d.ts +4 -0
- package/dist/api/logging-stream-registry.js +61 -0
- package/dist/api/logging-stream.d.ts +23 -0
- package/dist/api/logging-stream.js +36 -0
- package/dist/api/ocr.d.ts +30 -0
- package/dist/api/ocr.js +88 -0
- package/dist/api/parse-input.d.ts +2 -0
- package/dist/api/parse-input.js +14 -0
- package/dist/api/provide.d.ts +23 -0
- package/dist/api/provide.js +32 -0
- package/dist/api/rag.d.ts +251 -0
- package/dist/api/rag.js +485 -0
- package/dist/api/registry.d.ts +39 -0
- package/dist/api/registry.js +63 -0
- package/dist/api/resume.d.ts +29 -0
- package/dist/api/resume.js +36 -0
- package/dist/api/state.d.ts +22 -0
- package/dist/api/state.js +29 -0
- package/dist/api/stop-provider.d.ts +20 -0
- package/dist/api/stop-provider.js +30 -0
- package/dist/api/subscribe-logs.d.ts +27 -0
- package/dist/api/subscribe-logs.js +43 -0
- package/dist/api/suspend.d.ts +30 -0
- package/dist/api/suspend.js +37 -0
- package/dist/api/text-to-speech.d.ts +64 -0
- package/dist/api/text-to-speech.js +434 -0
- package/dist/api/transcribe.d.ts +75 -0
- package/dist/api/transcribe.js +256 -0
- package/dist/api/translate.d.ts +47 -0
- package/dist/api/translate.js +115 -0
- package/dist/api/unload-model.d.ts +17 -0
- package/dist/api/unload-model.js +42 -0
- package/dist/api/upscale.d.ts +44 -0
- package/dist/api/upscale.js +99 -0
- package/dist/api/video.d.ts +86 -0
- package/dist/api/video.js +176 -0
- package/dist/api/vla-helpers.d.ts +43 -0
- package/dist/api/vla-helpers.js +124 -0
- package/dist/api/vla.d.ts +68 -0
- package/dist/api/vla.js +113 -0
- package/dist/config/config-utils.d.ts +4 -0
- package/dist/config/config-utils.js +18 -0
- package/dist/config/resolve-config.d.ts +8 -0
- package/dist/config/resolve-config.js +74 -0
- package/dist/constants/audio.d.ts +1 -0
- package/dist/constants/audio.js +1 -0
- package/dist/constants/index.d.ts +2 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/registry.d.ts +1 -0
- package/dist/constants/registry.js +1 -0
- package/dist/dispatch.d.ts +18 -0
- package/dist/dispatch.js +271 -0
- package/dist/engine.d.ts +5 -0
- package/dist/engine.js +27 -0
- package/dist/errors/index.d.ts +416 -0
- package/dist/errors/index.js +681 -0
- package/dist/errors/options.d.ts +2 -0
- package/dist/errors/options.js +13 -0
- package/dist/handlers/cancelHandler.d.ts +29 -0
- package/dist/handlers/cancelHandler.js +67 -0
- package/dist/handlers/completion-orchestrate.d.ts +41 -0
- package/dist/handlers/completion-orchestrate.js +190 -0
- package/dist/handlers/delete-cache.d.ts +11 -0
- package/dist/handlers/delete-cache.js +37 -0
- package/dist/handlers/download-asset.d.ts +2 -0
- package/dist/handlers/download-asset.js +146 -0
- package/dist/handlers/finetune.d.ts +2 -0
- package/dist/handlers/finetune.js +18 -0
- package/dist/handlers/get-loaded-model-info.d.ts +2 -0
- package/dist/handlers/get-loaded-model-info.js +45 -0
- package/dist/handlers/get-model-info.d.ts +2 -0
- package/dist/handlers/get-model-info.js +173 -0
- package/dist/handlers/get-system-resources.d.ts +2 -0
- package/dist/handlers/get-system-resources.js +53 -0
- package/dist/handlers/heartbeat.d.ts +2 -0
- package/dist/handlers/heartbeat.js +3 -0
- package/dist/handlers/load-model/download-manager.d.ts +95 -0
- package/dist/handlers/load-model/download-manager.js +306 -0
- package/dist/handlers/load-model/download-stats.d.ts +7 -0
- package/dist/handlers/load-model/download-stats.js +126 -0
- package/dist/handlers/load-model/handler.d.ts +2 -0
- package/dist/handlers/load-model/handler.js +287 -0
- package/dist/handlers/load-model/http.d.ts +3 -0
- package/dist/handlers/load-model/http.js +857 -0
- package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
- package/dist/handlers/load-model/hyperdrive.js +560 -0
- package/dist/handlers/load-model/index.d.ts +2 -0
- package/dist/handlers/load-model/index.js +2 -0
- package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/handlers/load-model/registry-client-options.js +18 -0
- package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/handlers/load-model/registry-companion-set.js +206 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
- package/dist/handlers/load-model/registry-download-utils.js +112 -0
- package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
- package/dist/handlers/load-model/registry-metadata.js +23 -0
- package/dist/handlers/load-model/registry.d.ts +4 -0
- package/dist/handlers/load-model/registry.js +298 -0
- package/dist/handlers/load-model/resolve-session.d.ts +33 -0
- package/dist/handlers/load-model/resolve-session.js +62 -0
- package/dist/handlers/load-model/resolve.d.ts +5 -0
- package/dist/handlers/load-model/resolve.js +158 -0
- package/dist/handlers/load-model/types.d.ts +59 -0
- package/dist/handlers/load-model/types.js +28 -0
- package/dist/handlers/logging-stream.d.ts +2 -0
- package/dist/handlers/logging-stream.js +39 -0
- package/dist/handlers/plugin-dispatch.d.ts +3 -0
- package/dist/handlers/plugin-dispatch.js +63 -0
- package/dist/handlers/plugin-invoke.d.ts +3 -0
- package/dist/handlers/plugin-invoke.js +81 -0
- package/dist/handlers/registry.d.ts +4 -0
- package/dist/handlers/registry.js +156 -0
- package/dist/handlers/resume.d.ts +2 -0
- package/dist/handlers/resume.js +11 -0
- package/dist/handlers/state.d.ts +2 -0
- package/dist/handlers/state.js +7 -0
- package/dist/handlers/suspend.d.ts +2 -0
- package/dist/handlers/suspend.js +11 -0
- package/dist/handlers/types.d.ts +13 -0
- package/dist/handlers/types.js +1 -0
- package/dist/handlers/unload-model.d.ts +2 -0
- package/dist/handlers/unload-model.js +29 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +12 -0
- package/dist/logging/addon.d.ts +5 -0
- package/dist/logging/addon.js +75 -0
- package/dist/logging/index.d.ts +5 -0
- package/dist/logging/index.js +4 -0
- package/dist/logging/logger.d.ts +41 -0
- package/dist/logging/logger.js +209 -0
- package/dist/logging/namespaces.d.ts +6 -0
- package/dist/logging/namespaces.js +7 -0
- package/dist/logging/types.d.ts +37 -0
- package/dist/logging/types.js +1 -0
- package/dist/logging/utils.d.ts +5 -0
- package/dist/logging/utils.js +113 -0
- package/dist/models/registry/index.d.ts +18 -0
- package/dist/models/registry/index.js +21 -0
- package/dist/models/registry/models.d.ts +22680 -0
- package/dist/models/registry/models.js +23908 -0
- package/dist/p2p/cancel-delegated.d.ts +3 -0
- package/dist/p2p/cancel-delegated.js +64 -0
- package/dist/p2p/completion-stream-delegated.d.ts +4 -0
- package/dist/p2p/completion-stream-delegated.js +51 -0
- package/dist/p2p/delegate-client.d.ts +9 -0
- package/dist/p2p/delegate-client.js +219 -0
- package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
- package/dist/p2p/delegate-connect-diagnostics.js +38 -0
- package/dist/p2p/delegate-transport.d.ts +18 -0
- package/dist/p2p/delegate-transport.js +237 -0
- package/dist/p2p/error-handlers.d.ts +4 -0
- package/dist/p2p/error-handlers.js +31 -0
- package/dist/p2p/handle-request.d.ts +2 -0
- package/dist/p2p/handle-request.js +171 -0
- package/dist/p2p/handler-utils.d.ts +19 -0
- package/dist/p2p/handler-utils.js +180 -0
- package/dist/p2p/heartbeat-delegated.d.ts +3 -0
- package/dist/p2p/heartbeat-delegated.js +32 -0
- package/dist/p2p/load-model-delegated.d.ts +6 -0
- package/dist/p2p/load-model-delegated.js +132 -0
- package/dist/p2p/progress-throttle.d.ts +7 -0
- package/dist/p2p/progress-throttle.js +47 -0
- package/dist/p2p/provideHandler/connection.d.ts +2 -0
- package/dist/p2p/provideHandler/connection.js +38 -0
- package/dist/p2p/provideHandler/handler.d.ts +2 -0
- package/dist/p2p/provideHandler/handler.js +66 -0
- package/dist/p2p/provideHandler/proxy.d.ts +2 -0
- package/dist/p2p/provideHandler/proxy.js +24 -0
- package/dist/p2p/provideHandler.d.ts +1 -0
- package/dist/p2p/provideHandler.js +2 -0
- package/dist/p2p/rpc-utils.d.ts +4 -0
- package/dist/p2p/rpc-utils.js +8 -0
- package/dist/p2p/stopProvideHandler.d.ts +2 -0
- package/dist/p2p/stopProvideHandler.js +29 -0
- package/dist/p2p/swarm.d.ts +9 -0
- package/dist/p2p/swarm.js +103 -0
- package/dist/p2p/transport-selector.d.ts +2 -0
- package/dist/p2p/transport-selector.js +18 -0
- package/dist/p2p/unload-model-delegated.d.ts +2 -0
- package/dist/p2p/unload-model-delegated.js +31 -0
- package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
- package/dist/plugins/builtin/asr-ggml/config.js +46 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
- package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
- package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
- package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
- package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
- package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
- package/dist/plugins/builtin/index.d.ts +11 -0
- package/dist/plugins/builtin/index.js +11 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
- package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
- package/dist/plugins/index.d.ts +9 -0
- package/dist/plugins/index.js +12 -0
- package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
- package/dist/plugins/ops/bci-transcribe.js +248 -0
- package/dist/plugins/ops/cancel.d.ts +25 -0
- package/dist/plugins/ops/cancel.js +43 -0
- package/dist/plugins/ops/embed.d.ts +15 -0
- package/dist/plugins/ops/embed.js +193 -0
- package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
- package/dist/plugins/ops/kv-cache-retention.js +121 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
- package/dist/plugins/ops/kv-cache-utils.js +120 -0
- package/dist/plugins/ops/load-model.d.ts +8 -0
- package/dist/plugins/ops/load-model.js +82 -0
- package/dist/plugins/ops/transcribe.d.ts +20 -0
- package/dist/plugins/ops/transcribe.js +343 -0
- package/dist/plugins/ops/translate.d.ts +7 -0
- package/dist/plugins/ops/translate.js +256 -0
- package/dist/plugins/ops/unload-model.d.ts +2 -0
- package/dist/plugins/ops/unload-model.js +45 -0
- package/dist/plugins/registry.d.ts +9 -0
- package/dist/plugins/registry.js +107 -0
- package/dist/profiling/aggregator.d.ts +14 -0
- package/dist/profiling/aggregator.js +100 -0
- package/dist/profiling/backend-diagnostics.d.ts +8 -0
- package/dist/profiling/backend-diagnostics.js +36 -0
- package/dist/profiling/clock.d.ts +8 -0
- package/dist/profiling/clock.js +23 -0
- package/dist/profiling/controller.d.ts +34 -0
- package/dist/profiling/controller.js +130 -0
- package/dist/profiling/delegation-profiler.d.ts +31 -0
- package/dist/profiling/delegation-profiler.js +155 -0
- package/dist/profiling/envelope.d.ts +10 -0
- package/dist/profiling/envelope.js +45 -0
- package/dist/profiling/events.d.ts +19 -0
- package/dist/profiling/events.js +35 -0
- package/dist/profiling/exporters.d.ts +9 -0
- package/dist/profiling/exporters.js +242 -0
- package/dist/profiling/index.d.ts +91 -0
- package/dist/profiling/index.js +87 -0
- package/dist/profiling/model-execution.d.ts +15 -0
- package/dist/profiling/model-execution.js +34 -0
- package/dist/profiling/operation-metrics.d.ts +24 -0
- package/dist/profiling/operation-metrics.js +370 -0
- package/dist/profiling/operation-wrappers.d.ts +17 -0
- package/dist/profiling/operation-wrappers.js +196 -0
- package/dist/profiling/profiler.d.ts +27 -0
- package/dist/profiling/profiler.js +127 -0
- package/dist/profiling/ring-buffer.d.ts +18 -0
- package/dist/profiling/ring-buffer.js +83 -0
- package/dist/profiling/types.d.ts +52 -0
- package/dist/profiling/types.js +1 -0
- package/dist/rag/handler.d.ts +2 -0
- package/dist/rag/handler.js +272 -0
- package/dist/rag/handlers/chunk.d.ts +2 -0
- package/dist/rag/handlers/chunk.js +7 -0
- package/dist/rag/handlers/close-workspace.d.ts +2 -0
- package/dist/rag/handlers/close-workspace.js +9 -0
- package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
- package/dist/rag/handlers/delete-embeddings.js +11 -0
- package/dist/rag/handlers/delete-workspace.d.ts +2 -0
- package/dist/rag/handlers/delete-workspace.js +14 -0
- package/dist/rag/handlers/ingest.d.ts +9 -0
- package/dist/rag/handlers/ingest.js +19 -0
- package/dist/rag/handlers/list-workspaces.d.ts +2 -0
- package/dist/rag/handlers/list-workspaces.js +4 -0
- package/dist/rag/handlers/reindex.d.ts +9 -0
- package/dist/rag/handlers/reindex.js +16 -0
- package/dist/rag/handlers/save-embeddings.d.ts +9 -0
- package/dist/rag/handlers/save-embeddings.js +15 -0
- package/dist/rag/handlers/search.d.ts +2 -0
- package/dist/rag/handlers/search.js +16 -0
- package/dist/rag/index.d.ts +11 -0
- package/dist/rag/index.js +11 -0
- package/dist/rag/rag-operation-manager.d.ts +28 -0
- package/dist/rag/rag-operation-manager.js +76 -0
- package/dist/rag/rag-workspace-manager.d.ts +14 -0
- package/dist/rag/rag-workspace-manager.js +143 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +166 -0
- package/dist/resources/collector.d.ts +643 -0
- package/dist/resources/collector.js +226 -0
- package/dist/resources/instance.d.ts +4 -0
- package/dist/resources/instance.js +15 -0
- package/dist/resources/native.d.ts +2 -0
- package/dist/resources/native.js +35 -0
- package/dist/resources/normalize.d.ts +12 -0
- package/dist/resources/normalize.js +73 -0
- package/dist/resources/types.d.ts +65 -0
- package/dist/resources/types.js +1 -0
- package/dist/runtime/cache-lock.d.ts +2 -0
- package/dist/runtime/cache-lock.js +67 -0
- package/dist/runtime/disposable-scope.d.ts +35 -0
- package/dist/runtime/disposable-scope.js +69 -0
- package/dist/runtime/env.d.ts +16 -0
- package/dist/runtime/env.js +53 -0
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +20 -0
- package/dist/runtime/lifecycle.js +102 -0
- package/dist/runtime/logging-stream-registry.d.ts +27 -0
- package/dist/runtime/logging-stream-registry.js +133 -0
- package/dist/runtime/model-config-utils.d.ts +14 -0
- package/dist/runtime/model-config-utils.js +128 -0
- package/dist/runtime/model-registry.d.ts +74 -0
- package/dist/runtime/model-registry.js +130 -0
- package/dist/runtime/registry-client.d.ts +4 -0
- package/dist/runtime/registry-client.js +118 -0
- package/dist/runtime/request-context.d.ts +72 -0
- package/dist/runtime/request-context.js +114 -0
- package/dist/runtime/request-id.d.ts +17 -0
- package/dist/runtime/request-id.js +29 -0
- package/dist/runtime/request-registry.d.ts +194 -0
- package/dist/runtime/request-registry.js +546 -0
- package/dist/runtime/runtime-lifecycle.d.ts +37 -0
- package/dist/runtime/runtime-lifecycle.js +163 -0
- package/dist/runtime/state.d.ts +26 -0
- package/dist/runtime/state.js +138 -0
- package/dist/schemas/audio-gen.d.ts +252 -0
- package/dist/schemas/audio-gen.js +69 -0
- package/dist/schemas/batch-completion-stream.d.ts +467 -0
- package/dist/schemas/batch-completion-stream.js +123 -0
- package/dist/schemas/bci-config.d.ts +74 -0
- package/dist/schemas/bci-config.js +58 -0
- package/dist/schemas/bci.d.ts +178 -0
- package/dist/schemas/bci.js +89 -0
- package/dist/schemas/cancel.d.ts +203 -0
- package/dist/schemas/cancel.js +135 -0
- package/dist/schemas/classification.d.ts +52 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +6420 -0
- package/dist/schemas/common.js +133 -0
- package/dist/schemas/completion-event.d.ts +248 -0
- package/dist/schemas/completion-event.js +94 -0
- package/dist/schemas/completion-stream.d.ts +475 -0
- package/dist/schemas/completion-stream.js +236 -0
- package/dist/schemas/config.d.ts +5022 -0
- package/dist/schemas/config.js +169 -0
- package/dist/schemas/delegate.d.ts +29 -0
- package/dist/schemas/delegate.js +40 -0
- package/dist/schemas/download-asset.d.ts +225 -0
- package/dist/schemas/download-asset.js +71 -0
- package/dist/schemas/embed.d.ts +41 -0
- package/dist/schemas/embed.js +35 -0
- package/dist/schemas/engine-addon-map.d.ts +20 -0
- package/dist/schemas/engine-addon-map.js +78 -0
- package/dist/schemas/error.d.ts +40 -0
- package/dist/schemas/error.js +65 -0
- package/dist/schemas/errors.d.ts +119 -0
- package/dist/schemas/errors.js +615 -0
- package/dist/schemas/finetune.d.ts +621 -0
- package/dist/schemas/finetune.js +163 -0
- package/dist/schemas/index.d.ts +45 -0
- package/dist/schemas/index.js +45 -0
- package/dist/schemas/json-schema.d.ts +15 -0
- package/dist/schemas/json-schema.js +2 -0
- package/dist/schemas/lifecycle.d.ts +38 -0
- package/dist/schemas/lifecycle.js +21 -0
- package/dist/schemas/llamacpp-config.d.ts +403 -0
- package/dist/schemas/llamacpp-config.js +133 -0
- package/dist/schemas/load-model.d.ts +25635 -0
- package/dist/schemas/load-model.js +629 -0
- package/dist/schemas/logging-stream.d.ts +33 -0
- package/dist/schemas/logging-stream.js +17 -0
- package/dist/schemas/mcp-adapter.d.ts +46 -0
- package/dist/schemas/mcp-adapter.js +1 -0
- package/dist/schemas/model-file.d.ts +27 -0
- package/dist/schemas/model-file.js +29 -0
- package/dist/schemas/model-info.d.ts +197 -0
- package/dist/schemas/model-info.js +144 -0
- package/dist/schemas/model-ops.d.ts +36 -0
- package/dist/schemas/model-ops.js +40 -0
- package/dist/schemas/model-src-utils.d.ts +192 -0
- package/dist/schemas/model-src-utils.js +61 -0
- package/dist/schemas/model-types.d.ts +275 -0
- package/dist/schemas/model-types.js +223 -0
- package/dist/schemas/ocr.d.ts +138 -0
- package/dist/schemas/ocr.js +59 -0
- package/dist/schemas/path-validation.d.ts +9 -0
- package/dist/schemas/path-validation.js +22 -0
- package/dist/schemas/plugin.d.ts +341 -0
- package/dist/schemas/plugin.js +241 -0
- package/dist/schemas/profiling.d.ts +695 -0
- package/dist/schemas/profiling.js +100 -0
- package/dist/schemas/provide.d.ts +50 -0
- package/dist/schemas/provide.js +32 -0
- package/dist/schemas/rag.d.ts +301 -0
- package/dist/schemas/rag.js +211 -0
- package/dist/schemas/registry.d.ts +453 -0
- package/dist/schemas/registry.js +104 -0
- package/dist/schemas/reload-config.d.ts +585 -0
- package/dist/schemas/reload-config.js +42 -0
- package/dist/schemas/runtime-context.d.ts +18 -0
- package/dist/schemas/runtime-context.js +7 -0
- package/dist/schemas/sdcpp-config.d.ts +1126 -0
- package/dist/schemas/sdcpp-config.js +748 -0
- package/dist/schemas/system-resources.d.ts +3966 -0
- package/dist/schemas/system-resources.js +149 -0
- package/dist/schemas/text-to-speech.d.ts +1352 -0
- package/dist/schemas/text-to-speech.js +364 -0
- package/dist/schemas/tools.d.ts +80 -0
- package/dist/schemas/tools.js +52 -0
- package/dist/schemas/transcription-config.d.ts +161 -0
- package/dist/schemas/transcription-config.js +156 -0
- package/dist/schemas/transcription.d.ts +261 -0
- package/dist/schemas/transcription.js +145 -0
- package/dist/schemas/translate.d.ts +138 -0
- package/dist/schemas/translate.js +103 -0
- package/dist/schemas/translation-config.d.ts +1031 -0
- package/dist/schemas/translation-config.js +170 -0
- package/dist/schemas/vla.d.ts +110 -0
- package/dist/schemas/vla.js +151 -0
- package/dist/selection.d.ts +8 -0
- package/dist/selection.js +14 -0
- package/dist/surface.d.ts +18 -0
- package/dist/surface.js +30 -0
- package/dist/utils/addon-responses.d.ts +58 -0
- package/dist/utils/addon-responses.js +1 -0
- package/dist/utils/aggregate-events.d.ts +25 -0
- package/dist/utils/aggregate-events.js +73 -0
- package/dist/utils/archive.d.ts +4 -0
- package/dist/utils/archive.js +180 -0
- package/dist/utils/asr-events.d.ts +14 -0
- package/dist/utils/asr-events.js +30 -0
- package/dist/utils/audio/decoder.d.ts +4 -0
- package/dist/utils/audio/decoder.js +80 -0
- package/dist/utils/audio-input.d.ts +9 -0
- package/dist/utils/audio-input.js +34 -0
- package/dist/utils/cache/auto-history.d.ts +3 -0
- package/dist/utils/cache/auto-history.js +26 -0
- package/dist/utils/cache/paths.d.ts +42 -0
- package/dist/utils/cache/paths.js +133 -0
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/cache/types.js +1 -0
- package/dist/utils/cache-normalize.d.ts +9 -0
- package/dist/utils/cache-normalize.js +20 -0
- package/dist/utils/checksum.d.ts +3 -0
- package/dist/utils/checksum.js +29 -0
- package/dist/utils/clear-storage.d.ts +4 -0
- package/dist/utils/clear-storage.js +14 -0
- package/dist/utils/completion-normalizer.d.ts +17 -0
- package/dist/utils/completion-normalizer.js +453 -0
- package/dist/utils/config-transform.d.ts +2 -0
- package/dist/utils/config-transform.js +21 -0
- package/dist/utils/decorate-promise.d.ts +33 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/encoding.d.ts +16 -0
- package/dist/utils/encoding.js +27 -0
- package/dist/utils/expand-gguf-shards.d.ts +2 -0
- package/dist/utils/expand-gguf-shards.js +22 -0
- package/dist/utils/formatting.d.ts +18 -0
- package/dist/utils/formatting.js +56 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
- package/dist/utils/gguf-tensor-extractor.js +311 -0
- package/dist/utils/global-singleton.d.ts +1 -0
- package/dist/utils/global-singleton.js +10 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/load-model-validation.d.ts +6 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/dist/utils/mcp-adapter.d.ts +11 -0
- package/dist/utils/mcp-adapter.js +96 -0
- package/dist/utils/multi-gpu-mobile.d.ts +4 -0
- package/dist/utils/multi-gpu-mobile.js +10 -0
- package/dist/utils/object.d.ts +1 -0
- package/dist/utils/object.js +3 -0
- package/dist/utils/path-security.d.ts +22 -0
- package/dist/utils/path-security.js +69 -0
- package/dist/utils/qvac-paths.d.ts +1 -0
- package/dist/utils/qvac-paths.js +5 -0
- package/dist/utils/response-format.d.ts +2 -0
- package/dist/utils/response-format.js +20 -0
- package/dist/utils/shard-utils.d.ts +47 -0
- package/dist/utils/shard-utils.js +123 -0
- package/dist/utils/tool-helpers.d.ts +24 -0
- package/dist/utils/tool-helpers.js +111 -0
- package/dist/utils/tool-integration.d.ts +23 -0
- package/dist/utils/tool-integration.js +29 -0
- package/dist/utils/tools/dialect.d.ts +7 -0
- package/dist/utils/tools/dialect.js +20 -0
- package/dist/utils/tools/index.d.ts +2 -0
- package/dist/utils/tools/index.js +2 -0
- package/dist/utils/tools/parser.d.ts +5 -0
- package/dist/utils/tools/parser.js +69 -0
- package/dist/utils/tools/parsers/dsml.d.ts +3 -0
- package/dist/utils/tools/parsers/dsml.js +177 -0
- package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
- package/dist/utils/tools/parsers/gemma4native.js +65 -0
- package/dist/utils/tools/parsers/harmony.d.ts +3 -0
- package/dist/utils/tools/parsers/harmony.js +57 -0
- package/dist/utils/tools/parsers/hermes.d.ts +3 -0
- package/dist/utils/tools/parsers/hermes.js +122 -0
- package/dist/utils/tools/parsers/json.d.ts +5 -0
- package/dist/utils/tools/parsers/json.js +115 -0
- package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
- package/dist/utils/tools/parsers/pythonic.js +267 -0
- package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
- package/dist/utils/tools/parsers/qwen35.js +125 -0
- package/dist/utils/tools/shared.d.ts +17 -0
- package/dist/utils/tools/shared.js +52 -0
- package/dist/utils/transcribe-metadata.d.ts +20 -0
- package/dist/utils/transcribe-metadata.js +28 -0
- package/dist/utils/tts-stats.d.ts +25 -0
- package/dist/utils/tts-stats.js +16 -0
- package/dist/utils/withTimeout.d.ts +17 -0
- package/dist/utils/withTimeout.js +47 -0
- package/dist/utils/zod-error.d.ts +2 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +266 -1
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
53
|
+
import { getModel } from '../../runtime/model-registry.js';
|
|
54
|
+
import Buffer from 'bare-buffer';
|
|
55
|
+
import { getEngineLogger } from '../../logging/index.js';
|
|
56
|
+
import { TranscriptionFailedError } from '../../errors/index.js';
|
|
57
|
+
import { nowMs } from '../../profiling/index.js';
|
|
58
|
+
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
59
|
+
import { toTranscribeSegment } from '../../utils/transcribe-metadata.js';
|
|
60
|
+
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
61
|
+
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
62
|
+
async function runBci(model, input) {
|
|
63
|
+
switch (input.type) {
|
|
64
|
+
case 'base64':
|
|
65
|
+
return model.transcribe(Buffer.from(input.value, 'base64'));
|
|
66
|
+
case 'filePath':
|
|
67
|
+
return model.transcribeFile(input.value);
|
|
68
|
+
default:
|
|
69
|
+
throw new TranscriptionFailedError('Invalid neural input');
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
export async function* bciTranscribe(params, requestId) {
|
|
73
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
74
|
+
try {
|
|
75
|
+
const { modelId, metadata } = params;
|
|
76
|
+
// Open a request-scoped lifecycle. The registry routes
|
|
77
|
+
// `cancel({ requestId })` and `cancel({ modelId, kind: "transcribe" })`
|
|
78
|
+
// through this context. Falls back to a generated id if the
|
|
79
|
+
// caller didn't send one.
|
|
80
|
+
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
81
|
+
requestId: requestId ?? generateRandomRequestId(),
|
|
82
|
+
kind: 'transcribe',
|
|
83
|
+
modelId
|
|
84
|
+
}), true);
|
|
85
|
+
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
86
|
+
const model = getModel(modelId);
|
|
87
|
+
// Hard-cancel wiring: the BCI addon exposes a model-wide `cancel()`
|
|
88
|
+
// that interrupts the currently-running job. The per-iteration
|
|
89
|
+
// `if (ctx.signal.aborted) break` below is the soft-cancel safety net
|
|
90
|
+
// for the gap between the abort firing and the iterator's next pull.
|
|
91
|
+
const onAbort = () => {
|
|
92
|
+
if (typeof model.cancel === 'function') {
|
|
93
|
+
model.cancel().catch((err) => {
|
|
94
|
+
requestLogger.warn(`[cancel] model.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
99
|
+
if (ctx.signal.aborted)
|
|
100
|
+
onAbort();
|
|
101
|
+
ctx.scope.defer(() => {
|
|
102
|
+
ctx.signal.removeEventListener('abort', onAbort);
|
|
103
|
+
});
|
|
104
|
+
const modelStart = nowMs();
|
|
105
|
+
const response = await runBci(model, params.neuralData);
|
|
106
|
+
for await (const output of response.iterate()) {
|
|
107
|
+
if (ctx.signal.aborted)
|
|
108
|
+
break;
|
|
109
|
+
requestLogger.debug('BCI Transcription Update:', output);
|
|
110
|
+
const chunks = Array.isArray(output) ? output : [output];
|
|
111
|
+
if (metadata) {
|
|
112
|
+
for (const chunk of chunks) {
|
|
113
|
+
if (!chunk.text)
|
|
114
|
+
continue;
|
|
115
|
+
yield toTranscribeSegment(chunk);
|
|
116
|
+
}
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
const text = chunks.map((chunk) => chunk.text).join('');
|
|
120
|
+
if (text.trim()) {
|
|
121
|
+
yield text;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
const modelExecutionMs = nowMs() - modelStart;
|
|
125
|
+
const stats = {
|
|
126
|
+
...(response.stats?.audioDurationMs !== undefined && {
|
|
127
|
+
audioDuration: response.stats.audioDurationMs
|
|
128
|
+
}),
|
|
129
|
+
...(response.stats?.realTimeFactor !== undefined && {
|
|
130
|
+
realTimeFactor: response.stats.realTimeFactor
|
|
131
|
+
}),
|
|
132
|
+
...(response.stats?.tokensPerSecond !== undefined && {
|
|
133
|
+
tokensPerSecond: response.stats.tokensPerSecond
|
|
134
|
+
}),
|
|
135
|
+
...(response.stats?.totalTokens !== undefined && {
|
|
136
|
+
totalTokens: response.stats.totalTokens
|
|
137
|
+
}),
|
|
138
|
+
...(response.stats?.totalSegments !== undefined && {
|
|
139
|
+
totalSegments: response.stats.totalSegments
|
|
140
|
+
}),
|
|
141
|
+
...(response.stats?.whisperEncodeMs !== undefined && {
|
|
142
|
+
whisperEncodeTime: response.stats.whisperEncodeMs
|
|
143
|
+
}),
|
|
144
|
+
...(response.stats?.whisperDecodeMs !== undefined && {
|
|
145
|
+
whisperDecodeTime: response.stats.whisperDecodeMs
|
|
146
|
+
}),
|
|
147
|
+
...(response.stats?.encoderMs !== undefined && {
|
|
148
|
+
encoderTime: response.stats.encoderMs
|
|
149
|
+
}),
|
|
150
|
+
...(response.stats?.decoderMs !== undefined && {
|
|
151
|
+
decoderTime: response.stats.decoderMs
|
|
152
|
+
}),
|
|
153
|
+
...(response.stats?.melSpecMs !== undefined && {
|
|
154
|
+
melSpecTime: response.stats.melSpecMs
|
|
155
|
+
}),
|
|
156
|
+
...(response.stats?.backendDevice !== undefined && {
|
|
157
|
+
backendDevice: response.stats.backendDevice
|
|
158
|
+
}),
|
|
159
|
+
...(response.stats?.backendId !== undefined && {
|
|
160
|
+
backendId: response.stats.backendId
|
|
161
|
+
}),
|
|
162
|
+
...(response.stats?.gpuMemTotalMb !== undefined && {
|
|
163
|
+
gpuMemTotalMb: response.stats.gpuMemTotalMb
|
|
164
|
+
}),
|
|
165
|
+
...(response.stats?.gpuMemFreeMb !== undefined && {
|
|
166
|
+
gpuMemFreeMb: response.stats.gpuMemFreeMb
|
|
167
|
+
})
|
|
168
|
+
};
|
|
169
|
+
return buildStreamResult(modelExecutionMs, stats);
|
|
170
|
+
}
|
|
171
|
+
catch (e_1) {
|
|
172
|
+
env_1.error = e_1;
|
|
173
|
+
env_1.hasError = true;
|
|
174
|
+
}
|
|
175
|
+
finally {
|
|
176
|
+
const result_1 = __disposeResources(env_1);
|
|
177
|
+
if (result_1)
|
|
178
|
+
await result_1;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts, requestId) {
|
|
182
|
+
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
183
|
+
try {
|
|
184
|
+
// Same `kind: "transcribe"` as the unary BCI variant — the registry
|
|
185
|
+
// doesn't distinguish streaming vs non-streaming variants of the same
|
|
186
|
+
// operation, so `cancel({ modelId, kind: "transcribe" })` cancels
|
|
187
|
+
// either shape.
|
|
188
|
+
const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
|
|
189
|
+
requestId: requestId ?? generateRandomRequestId(),
|
|
190
|
+
kind: 'transcribe',
|
|
191
|
+
modelId
|
|
192
|
+
}), true);
|
|
193
|
+
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
194
|
+
const model = getModel(modelId);
|
|
195
|
+
// Hard-cancel wiring: `cancel()` tears down the active stream driver
|
|
196
|
+
// and interrupts the running window job. The per-iteration abort guard
|
|
197
|
+
// below is the soft-cancel safety net between the abort firing and the
|
|
198
|
+
// iterator's next pull.
|
|
199
|
+
const onAbort = () => {
|
|
200
|
+
if (typeof model.cancel === 'function') {
|
|
201
|
+
model.cancel().catch((err) => {
|
|
202
|
+
requestLogger.warn(`[cancel] model.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
207
|
+
if (ctx.signal.aborted)
|
|
208
|
+
onAbort();
|
|
209
|
+
ctx.scope.defer(() => {
|
|
210
|
+
ctx.signal.removeEventListener('abort', onAbort);
|
|
211
|
+
});
|
|
212
|
+
const streamOpts = {
|
|
213
|
+
...(opts?.windowTimesteps !== undefined && {
|
|
214
|
+
windowTimesteps: opts.windowTimesteps
|
|
215
|
+
}),
|
|
216
|
+
...(opts?.hopTimesteps !== undefined && { hopTimesteps: opts.hopTimesteps }),
|
|
217
|
+
...(opts?.emit !== undefined && { emit: opts.emit })
|
|
218
|
+
};
|
|
219
|
+
const response = await model.transcribeStream(neuralStream, streamOpts);
|
|
220
|
+
for await (const output of response.iterate()) {
|
|
221
|
+
if (ctx.signal.aborted)
|
|
222
|
+
break;
|
|
223
|
+
requestLogger.debug('BCI Stream Transcription Update:', output);
|
|
224
|
+
const chunks = Array.isArray(output) ? output : [output];
|
|
225
|
+
if (metadata) {
|
|
226
|
+
for (const chunk of chunks) {
|
|
227
|
+
if (!chunk.text)
|
|
228
|
+
continue;
|
|
229
|
+
yield toTranscribeSegment(chunk);
|
|
230
|
+
}
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
const text = chunks.map((chunk) => chunk.text).join('');
|
|
234
|
+
if (text.trim()) {
|
|
235
|
+
yield text;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
catch (e_2) {
|
|
240
|
+
env_2.error = e_2;
|
|
241
|
+
env_2.hasError = true;
|
|
242
|
+
}
|
|
243
|
+
finally {
|
|
244
|
+
const result_2 = __disposeResources(env_2);
|
|
245
|
+
if (result_2)
|
|
246
|
+
await result_2;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type CancelInferenceBaseParams } from '../../schemas/index.js';
|
|
2
|
+
import type { RequestKind } from '../../runtime/index.js';
|
|
3
|
+
/**
|
|
4
|
+
* Broad cancel: abort every in-flight request matching `modelId` (and
|
|
5
|
+
* optionally a `kind`). Maps onto `RequestRegistry.cancel({ modelId })`
|
|
6
|
+
* — the registry walks active contexts and aborts each one's signal,
|
|
7
|
+
* which each handler has wired to its own addon-level / async unwind
|
|
8
|
+
* via the registry's `await using ctx = registry.begin(...)` block.
|
|
9
|
+
*
|
|
10
|
+
* Kept as a stable surface alongside the new `cancel({ requestId })`
|
|
11
|
+
* path: the caller may not have a `requestId` to hand (model unload,
|
|
12
|
+
* app shutdown, admin sweeps), and the escape hatch is cheap because
|
|
13
|
+
* the registry already does the matching.
|
|
14
|
+
*
|
|
15
|
+
* Returns the number of contexts whose abort was triggered by *this*
|
|
16
|
+
* call (already-cancelled contexts are skipped so callers can rely on
|
|
17
|
+
* the count to log "n requests cancelled" once). Used by the cancel
|
|
18
|
+
* handler to populate `CancelResponse.cancelled` and by internal
|
|
19
|
+
* callers that want to know whether anything landed.
|
|
20
|
+
*
|
|
21
|
+
* The function does exactly one thing — a registry walk.
|
|
22
|
+
*/
|
|
23
|
+
export declare function cancel(params: CancelInferenceBaseParams, opts?: {
|
|
24
|
+
kind?: RequestKind;
|
|
25
|
+
}): number;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { getModel } from '../../runtime/model-registry.js';
|
|
2
|
+
import { cancelInferenceBaseSchema } from '../../schemas/index.js';
|
|
3
|
+
import { ModelNotLoadedError } from '../../errors/index.js';
|
|
4
|
+
import { getRequestRegistry } from '../../runtime/index.js';
|
|
5
|
+
import { getEngineLogger } from '../../logging/index.js';
|
|
6
|
+
const logger = getEngineLogger();
|
|
7
|
+
/**
|
|
8
|
+
* Broad cancel: abort every in-flight request matching `modelId` (and
|
|
9
|
+
* optionally a `kind`). Maps onto `RequestRegistry.cancel({ modelId })`
|
|
10
|
+
* — the registry walks active contexts and aborts each one's signal,
|
|
11
|
+
* which each handler has wired to its own addon-level / async unwind
|
|
12
|
+
* via the registry's `await using ctx = registry.begin(...)` block.
|
|
13
|
+
*
|
|
14
|
+
* Kept as a stable surface alongside the new `cancel({ requestId })`
|
|
15
|
+
* path: the caller may not have a `requestId` to hand (model unload,
|
|
16
|
+
* app shutdown, admin sweeps), and the escape hatch is cheap because
|
|
17
|
+
* the registry already does the matching.
|
|
18
|
+
*
|
|
19
|
+
* Returns the number of contexts whose abort was triggered by *this*
|
|
20
|
+
* call (already-cancelled contexts are skipped so callers can rely on
|
|
21
|
+
* the count to log "n requests cancelled" once). Used by the cancel
|
|
22
|
+
* handler to populate `CancelResponse.cancelled` and by internal
|
|
23
|
+
* callers that want to know whether anything landed.
|
|
24
|
+
*
|
|
25
|
+
* The function does exactly one thing — a registry walk.
|
|
26
|
+
*/
|
|
27
|
+
export function cancel(params, opts) {
|
|
28
|
+
const { modelId } = cancelInferenceBaseSchema.parse(params);
|
|
29
|
+
const model = getModel(modelId);
|
|
30
|
+
if (!model) {
|
|
31
|
+
throw new ModelNotLoadedError(modelId);
|
|
32
|
+
}
|
|
33
|
+
const registry = getRequestRegistry();
|
|
34
|
+
const target = opts?.kind ? { modelId, kind: opts.kind } : { modelId };
|
|
35
|
+
const cancelled = registry.cancel(target);
|
|
36
|
+
if (cancelled === 0) {
|
|
37
|
+
// Callers (workbench "Stop" button, app shutdown sweeps) often
|
|
38
|
+
// fire-and-forget; log so operators can see when a broad cancel
|
|
39
|
+
// landed against a registry with nothing in flight on this model.
|
|
40
|
+
logger.debug(`[cancel] no in-flight request matched modelId=${modelId}${opts?.kind ? ` kind=${opts.kind}` : ''}`);
|
|
41
|
+
}
|
|
42
|
+
return cancelled;
|
|
43
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type EmbedParams, type EmbedStats } from '../../schemas/index.js';
|
|
2
|
+
export interface EmbedResult {
|
|
3
|
+
embedding: number[] | number[][];
|
|
4
|
+
stats?: EmbedStats;
|
|
5
|
+
}
|
|
6
|
+
export declare function embed(params: {
|
|
7
|
+
modelId: string;
|
|
8
|
+
text: string;
|
|
9
|
+
}, requestId?: string): Promise<EmbedResult>;
|
|
10
|
+
export declare function embed(params: {
|
|
11
|
+
modelId: string;
|
|
12
|
+
text: string[];
|
|
13
|
+
}, requestId?: string): Promise<EmbedResult>;
|
|
14
|
+
export declare function embed(params: EmbedParams, requestId?: string): Promise<EmbedResult>;
|
|
15
|
+
export declare function normalizeVector(vector: Float32Array): number[];
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
53
|
+
import { getModel } from '../../runtime/model-registry.js';
|
|
54
|
+
import { embedParamsSchema } from '../../schemas/index.js';
|
|
55
|
+
import { buildUnaryResult } from '../../profiling/model-execution.js';
|
|
56
|
+
import { EmbedNoEmbeddingsError, EmbedFailedError, InferenceCancelledError } from '../../errors/index.js';
|
|
57
|
+
import { nowMs } from '../../profiling/index.js';
|
|
58
|
+
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
59
|
+
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
60
|
+
import { getEngineLogger } from '../../logging/index.js';
|
|
61
|
+
export async function embed(params, requestId) {
|
|
62
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
63
|
+
try {
|
|
64
|
+
const { modelId, text } = embedParamsSchema.parse(params);
|
|
65
|
+
// Open a request-scoped lifecycle. The registry routes
|
|
66
|
+
// `cancel({ requestId })` and broad `cancel({ modelId, kind: "embeddings" })`
|
|
67
|
+
// straight to this context's signal. Falls back to a generated
|
|
68
|
+
// id if the caller didn't send one.
|
|
69
|
+
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
70
|
+
requestId: requestId ?? generateRandomRequestId(),
|
|
71
|
+
kind: 'embeddings',
|
|
72
|
+
modelId
|
|
73
|
+
})
|
|
74
|
+
// `requestLogger` is intentionally referenced once so the
|
|
75
|
+
// request-scoped logger is built at handler entry per the canonical
|
|
76
|
+
// shape, even when this op has no per-step emits beyond the
|
|
77
|
+
// registry's own lifecycle lines. Future addon-level warns inside
|
|
78
|
+
// this body should route through `requestLogger`.
|
|
79
|
+
, true);
|
|
80
|
+
// `requestLogger` is intentionally referenced once so the
|
|
81
|
+
// request-scoped logger is built at handler entry per the canonical
|
|
82
|
+
// shape, even when this op has no per-step emits beyond the
|
|
83
|
+
// registry's own lifecycle lines. Future addon-level warns inside
|
|
84
|
+
// this body should route through `requestLogger`.
|
|
85
|
+
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
86
|
+
const model = getModel(modelId);
|
|
87
|
+
// Hard-cancel wiring: llamacpp-embedding declares
|
|
88
|
+
// `cancel: { scope: "model", hard: true }`. The signal listener
|
|
89
|
+
// forwards an abort to the addon so C++ work stops as soon as the
|
|
90
|
+
// user cancels — mirrors the canonical handler shape documented in
|
|
91
|
+
// `request-lifecycle-primitives.mdc`. The post-await
|
|
92
|
+
// `signal.aborted` checks below are the soft-cancel safety net for
|
|
93
|
+
// the case where the abort fires between `model.run(...)` returning
|
|
94
|
+
// and `response.await()` resolving (both can take seconds on large
|
|
95
|
+
// batches).
|
|
96
|
+
const onAbort = () => {
|
|
97
|
+
const addon = model.addon;
|
|
98
|
+
if (addon?.cancel) {
|
|
99
|
+
addon.cancel.call(addon).catch((err) => {
|
|
100
|
+
requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
105
|
+
// `{ once: true }` does not fire if the signal is already aborted at
|
|
106
|
+
// register time. The registry may abort synchronously when a
|
|
107
|
+
// cancel-before-begin race resolves or the parent signal was pre-
|
|
108
|
+
// aborted — re-use `onAbort` so the listener body is the single
|
|
109
|
+
// source of truth for "what cancel does."
|
|
110
|
+
if (ctx.signal.aborted)
|
|
111
|
+
onAbort();
|
|
112
|
+
ctx.scope.defer(() => {
|
|
113
|
+
ctx.signal.removeEventListener('abort', onAbort);
|
|
114
|
+
});
|
|
115
|
+
const modelStart = nowMs();
|
|
116
|
+
const response = (await model.run(text));
|
|
117
|
+
if (ctx.signal.aborted) {
|
|
118
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
119
|
+
}
|
|
120
|
+
const rawEmbeddings = await response.await();
|
|
121
|
+
if (ctx.signal.aborted) {
|
|
122
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
123
|
+
}
|
|
124
|
+
const modelExecutionMs = nowMs() - modelStart;
|
|
125
|
+
const stats = {
|
|
126
|
+
...(response.stats?.total_time_ms !== undefined && { totalTime: response.stats.total_time_ms }),
|
|
127
|
+
...(response.stats?.tokens_per_second !== undefined && {
|
|
128
|
+
tokensPerSecond: response.stats.tokens_per_second
|
|
129
|
+
}),
|
|
130
|
+
...(response.stats?.total_tokens !== undefined && { totalTokens: response.stats.total_tokens }),
|
|
131
|
+
...(response.stats?.backendDevice !== undefined && {
|
|
132
|
+
backendDevice: response.stats.backendDevice
|
|
133
|
+
}),
|
|
134
|
+
...(response.stats?.context_size !== undefined && { contextSize: response.stats.context_size })
|
|
135
|
+
};
|
|
136
|
+
const embeddingsArray = rawEmbeddings[0];
|
|
137
|
+
if (Array.isArray(text)) {
|
|
138
|
+
if (!embeddingsArray || embeddingsArray.length === 0) {
|
|
139
|
+
throw new EmbedNoEmbeddingsError();
|
|
140
|
+
}
|
|
141
|
+
const embedding = embeddingsArray.map((embeddingVector) => {
|
|
142
|
+
if (!embeddingVector || embeddingVector.length === 0) {
|
|
143
|
+
throw new EmbedNoEmbeddingsError();
|
|
144
|
+
}
|
|
145
|
+
return normalizeVector(embeddingVector);
|
|
146
|
+
});
|
|
147
|
+
return buildUnaryResult({ embedding }, modelExecutionMs, stats);
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
const embeddingVector = embeddingsArray?.[0];
|
|
151
|
+
if (!embeddingVector || embeddingVector.length === 0) {
|
|
152
|
+
throw new EmbedNoEmbeddingsError();
|
|
153
|
+
}
|
|
154
|
+
return buildUnaryResult({ embedding: normalizeVector(embeddingVector) }, modelExecutionMs, stats);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
catch (e_1) {
|
|
158
|
+
env_1.error = e_1;
|
|
159
|
+
env_1.hasError = true;
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
const result_1 = __disposeResources(env_1);
|
|
163
|
+
if (result_1)
|
|
164
|
+
await result_1;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
export function normalizeVector(vector) {
|
|
168
|
+
let sumOfSquares = 0;
|
|
169
|
+
for (let i = 0; i < vector.length; i++) {
|
|
170
|
+
const value = vector[i];
|
|
171
|
+
if (!Number.isFinite(value)) {
|
|
172
|
+
throw new EmbedFailedError(`NormalizeVector: non-finite value at index ${i}: ${value}`);
|
|
173
|
+
}
|
|
174
|
+
sumOfSquares += value * value;
|
|
175
|
+
}
|
|
176
|
+
const magnitude = Math.sqrt(sumOfSquares);
|
|
177
|
+
const EPS_ZERO = 1e-12;
|
|
178
|
+
const UNIT_TOL = 1e-4;
|
|
179
|
+
// Handle bad norms
|
|
180
|
+
if (!Number.isFinite(magnitude) || magnitude < EPS_ZERO) {
|
|
181
|
+
return new Array(vector.length).fill(0);
|
|
182
|
+
}
|
|
183
|
+
// Early exit: already ~unit length
|
|
184
|
+
if (Math.abs(magnitude - 1) <= UNIT_TOL) {
|
|
185
|
+
return Array.from(vector);
|
|
186
|
+
}
|
|
187
|
+
const inverseMagnitude = 1 / magnitude;
|
|
188
|
+
const normalizedVector = new Array(vector.length);
|
|
189
|
+
for (let i = 0; i < vector.length; i++) {
|
|
190
|
+
normalizedVector[i] = vector[i] * inverseMagnitude;
|
|
191
|
+
}
|
|
192
|
+
return normalizedVector;
|
|
193
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface AutoCacheRetentionOptions {
|
|
2
|
+
activeCachePaths: string[];
|
|
3
|
+
maxBytes: number;
|
|
4
|
+
maxIdleMs: number;
|
|
5
|
+
nowMs: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function markAutoCacheKey(cacheKey: string): Promise<void>;
|
|
8
|
+
export declare function removeAutoCacheMarker(cacheKey: string): Promise<void>;
|
|
9
|
+
export declare function removeAutoCacheMarkerIfMissing(cacheKey: string): Promise<void>;
|
|
10
|
+
export declare function isCachePathWithinDirectory(directoryPath: string, cachePath: string): boolean;
|
|
11
|
+
export declare function planAutoCacheEvictions(options: AutoCacheRetentionOptions): Promise<string[]>;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { getKVCacheDir } from '../../utils/index.js';
|
|
4
|
+
import { getEngineLogger } from '../../logging/index.js';
|
|
5
|
+
const logger = getEngineLogger();
|
|
6
|
+
const AUTO_CACHE_MARKER_PREFIX = '.auto-cache-';
|
|
7
|
+
const AUTO_CACHE_MARKER_PATTERN = /^\.auto-cache-([a-f0-9]{16})$/;
|
|
8
|
+
function getAutoCacheMarkerPath(cacheKey) {
|
|
9
|
+
return path.join(getKVCacheDir(), `${AUTO_CACHE_MARKER_PREFIX}${cacheKey}`);
|
|
10
|
+
}
|
|
11
|
+
export async function markAutoCacheKey(cacheKey) {
|
|
12
|
+
try {
|
|
13
|
+
await fsPromises.writeFile(getAutoCacheMarkerPath(cacheKey), '');
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
logger.warn(`[kv-cache] Failed to write auto-cache retention marker. key=${cacheKey} error=${error instanceof Error ? error.message : String(error)}`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export async function removeAutoCacheMarker(cacheKey) {
|
|
20
|
+
try {
|
|
21
|
+
await fsPromises.unlink(getAutoCacheMarkerPath(cacheKey));
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
// The marker is best-effort cleanup metadata and may already be absent.
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export async function removeAutoCacheMarkerIfMissing(cacheKey) {
|
|
28
|
+
try {
|
|
29
|
+
await fsPromises.access(path.join(getKVCacheDir(), cacheKey));
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
await removeAutoCacheMarker(cacheKey);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export function isCachePathWithinDirectory(directoryPath, cachePath) {
|
|
36
|
+
const prefix = `${directoryPath}${path.sep}`;
|
|
37
|
+
return cachePath === directoryPath || cachePath.startsWith(prefix);
|
|
38
|
+
}
|
|
39
|
+
async function readDirectory(directoryPath) {
|
|
40
|
+
try {
|
|
41
|
+
return await fsPromises.readdir(directoryPath);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
async function inspectAutoCacheEntry(cacheKey, activeCachePaths) {
|
|
48
|
+
const directoryPath = path.join(getKVCacheDir(), cacheKey);
|
|
49
|
+
const modelDirectories = await readDirectory(directoryPath);
|
|
50
|
+
if (modelDirectories === null)
|
|
51
|
+
return null;
|
|
52
|
+
let size = 0;
|
|
53
|
+
let lastModifiedMs = 0;
|
|
54
|
+
for (const modelDirectory of modelDirectories) {
|
|
55
|
+
const modelPath = path.join(directoryPath, modelDirectory.toString());
|
|
56
|
+
const fileNames = await readDirectory(modelPath);
|
|
57
|
+
if (fileNames === null)
|
|
58
|
+
continue;
|
|
59
|
+
for (const fileName of fileNames) {
|
|
60
|
+
try {
|
|
61
|
+
const stats = await fsPromises.stat(path.join(modelPath, fileName.toString()));
|
|
62
|
+
if (!stats.isFile())
|
|
63
|
+
continue;
|
|
64
|
+
size += stats.size;
|
|
65
|
+
lastModifiedMs = Math.max(lastModifiedMs, stats.mtime.getTime());
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
// The cache may have moved or been rolled back while the sweep was inspecting it.
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
cacheKey,
|
|
74
|
+
directoryPath,
|
|
75
|
+
size,
|
|
76
|
+
lastModifiedMs,
|
|
77
|
+
active: activeCachePaths.some((cachePath) => isCachePathWithinDirectory(directoryPath, cachePath))
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
export async function planAutoCacheEvictions(options) {
|
|
81
|
+
const rootEntries = await readDirectory(getKVCacheDir());
|
|
82
|
+
if (rootEntries === null)
|
|
83
|
+
return [];
|
|
84
|
+
const entries = [];
|
|
85
|
+
for (const rootEntry of rootEntries) {
|
|
86
|
+
const markerMatch = AUTO_CACHE_MARKER_PATTERN.exec(rootEntry.toString());
|
|
87
|
+
if (markerMatch === null)
|
|
88
|
+
continue;
|
|
89
|
+
const cacheKey = markerMatch[1];
|
|
90
|
+
if (cacheKey === undefined)
|
|
91
|
+
continue;
|
|
92
|
+
const entry = await inspectAutoCacheEntry(cacheKey, options.activeCachePaths);
|
|
93
|
+
if (entry === null) {
|
|
94
|
+
await removeAutoCacheMarker(cacheKey);
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
entries.push(entry);
|
|
98
|
+
}
|
|
99
|
+
let retainedBytes = entries.reduce((total, entry) => total + entry.size, 0);
|
|
100
|
+
const evictions = new Set();
|
|
101
|
+
for (const entry of entries) {
|
|
102
|
+
if (entry.active)
|
|
103
|
+
continue;
|
|
104
|
+
const expired = entry.lastModifiedMs === 0 ||
|
|
105
|
+
(options.maxIdleMs > 0 && options.nowMs - entry.lastModifiedMs > options.maxIdleMs);
|
|
106
|
+
if (!expired)
|
|
107
|
+
continue;
|
|
108
|
+
evictions.add(entry.cacheKey);
|
|
109
|
+
retainedBytes -= entry.size;
|
|
110
|
+
}
|
|
111
|
+
const oldestFirst = entries
|
|
112
|
+
.filter((entry) => !entry.active && !evictions.has(entry.cacheKey))
|
|
113
|
+
.sort((a, b) => a.lastModifiedMs - b.lastModifiedMs);
|
|
114
|
+
for (const entry of oldestFirst) {
|
|
115
|
+
if (retainedBytes <= options.maxBytes)
|
|
116
|
+
break;
|
|
117
|
+
evictions.add(entry.cacheKey);
|
|
118
|
+
retainedBytes -= entry.size;
|
|
119
|
+
}
|
|
120
|
+
return Array.from(evictions);
|
|
121
|
+
}
|