@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -1,8 +1,63 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
53
|
+
import Buffer from 'bare-buffer';
|
|
1
54
|
import { toolCallbackResultSchema } from '../schemas/completion-stream.js';
|
|
2
55
|
import { aggregateEvents } from '../utils/aggregate-events.js';
|
|
3
|
-
import { CompletionFailedError } from '../errors/index.js';
|
|
4
|
-
import { dispatchPluginStream } from '
|
|
56
|
+
import { CompletionFailedError, InferenceCancelledError } from '../errors/index.js';
|
|
57
|
+
import { dispatchPluginStream } from '../handlers/plugin-dispatch.js';
|
|
5
58
|
import { executeToolSearch, TOOL_SEARCH_NAME } from '../utils/tools/defer.js';
|
|
59
|
+
import { getRequestRegistry } from '../runtime/request-context.js';
|
|
60
|
+
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
6
61
|
const DEFAULT_MAX_TOOL_TURNS = 8;
|
|
7
62
|
// Generous ceiling on one client-side tool execution: long enough for a slow
|
|
8
63
|
// MCP round trip, short enough that a client that stopped reading can't pin a
|
|
@@ -22,7 +77,8 @@ const MAX_TOOL_RESULT_LINE_BYTES = 16 * 1024 * 1024;
|
|
|
22
77
|
* and the reader stops holding the request stream open.
|
|
23
78
|
*/
|
|
24
79
|
export class ToolResultReader {
|
|
25
|
-
|
|
80
|
+
pending = [];
|
|
81
|
+
pendingBytes = 0;
|
|
26
82
|
results = new Map();
|
|
27
83
|
waiters = new Map();
|
|
28
84
|
failed;
|
|
@@ -38,25 +94,42 @@ export class ToolResultReader {
|
|
|
38
94
|
const next = await this.iterator.next();
|
|
39
95
|
if (next.done)
|
|
40
96
|
break;
|
|
41
|
-
|
|
97
|
+
const chunk = next.value;
|
|
98
|
+
let start = 0;
|
|
42
99
|
let nl;
|
|
43
|
-
while ((nl =
|
|
44
|
-
|
|
45
|
-
|
|
100
|
+
while ((nl = chunk.indexOf(0x0a, start)) !== -1) {
|
|
101
|
+
this.append(chunk.subarray(start, nl));
|
|
102
|
+
// A transport chunk can end inside a UTF-8 character. Decode complete lines only.
|
|
103
|
+
const line = Buffer.concat(this.pending).toString('utf8');
|
|
104
|
+
this.pending = [];
|
|
105
|
+
this.pendingBytes = 0;
|
|
46
106
|
if (line.trim())
|
|
47
107
|
this.accept(line);
|
|
108
|
+
start = nl + 1;
|
|
48
109
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
110
|
+
this.append(chunk.subarray(start));
|
|
111
|
+
}
|
|
112
|
+
if (!this.closed) {
|
|
113
|
+
const tail = Buffer.concat(this.pending).toString('utf8');
|
|
114
|
+
this.pending = [];
|
|
115
|
+
this.pendingBytes = 0;
|
|
116
|
+
if (tail.trim())
|
|
117
|
+
this.accept(tail);
|
|
52
118
|
}
|
|
53
|
-
if (!this.closed && this.buffer.trim())
|
|
54
|
-
this.accept(this.buffer);
|
|
55
119
|
}
|
|
56
120
|
catch (error) {
|
|
57
121
|
this.failWaiters(error instanceof Error ? error : new Error(String(error)));
|
|
58
122
|
}
|
|
59
123
|
}
|
|
124
|
+
append(bytes) {
|
|
125
|
+
if (this.pendingBytes + bytes.length > MAX_TOOL_RESULT_LINE_BYTES) {
|
|
126
|
+
throw new CompletionFailedError('tool result frame exceeds the maximum size');
|
|
127
|
+
}
|
|
128
|
+
if (bytes.length) {
|
|
129
|
+
this.pending.push(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
|
130
|
+
this.pendingBytes += bytes.length;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
60
133
|
failWaiters(error) {
|
|
61
134
|
this.failed = error;
|
|
62
135
|
// Wake every waiter; each rethrows via waitFor's failure check.
|
|
@@ -76,7 +149,12 @@ export class ToolResultReader {
|
|
|
76
149
|
this.results.set(parsed.callId, parsed);
|
|
77
150
|
}
|
|
78
151
|
}
|
|
79
|
-
|
|
152
|
+
/** Rejects with `InferenceCancelledError` when `signal` aborts before the result arrives. */
|
|
153
|
+
async waitFor(callId, timeoutMs = TOOL_RESULT_TIMEOUT_MS, signal) {
|
|
154
|
+
// A cancel that landed between turns aborts the signal before the wait
|
|
155
|
+
// starts; it wins over a result the client already sent.
|
|
156
|
+
if (signal?.aborted)
|
|
157
|
+
throw new InferenceCancelledError(callId);
|
|
80
158
|
const buffered = this.results.get(callId);
|
|
81
159
|
if (buffered) {
|
|
82
160
|
this.results.delete(callId);
|
|
@@ -85,12 +163,22 @@ export class ToolResultReader {
|
|
|
85
163
|
if (this.failed)
|
|
86
164
|
throw this.failed;
|
|
87
165
|
return await new Promise((resolve, reject) => {
|
|
88
|
-
const
|
|
166
|
+
const settle = () => {
|
|
167
|
+
clearTimeout(timer);
|
|
168
|
+
signal?.removeEventListener('abort', onAbort);
|
|
89
169
|
this.waiters.delete(callId);
|
|
170
|
+
};
|
|
171
|
+
const onAbort = () => {
|
|
172
|
+
settle();
|
|
173
|
+
reject(new InferenceCancelledError(callId));
|
|
174
|
+
};
|
|
175
|
+
const timer = setTimeout(() => {
|
|
176
|
+
settle();
|
|
90
177
|
reject(new CompletionFailedError(`tool callback ${callId} timed out`));
|
|
91
178
|
}, timeoutMs);
|
|
179
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
92
180
|
this.waiters.set(callId, (result) => {
|
|
93
|
-
|
|
181
|
+
settle();
|
|
94
182
|
resolve(result);
|
|
95
183
|
});
|
|
96
184
|
});
|
|
@@ -104,7 +192,8 @@ export class ToolResultReader {
|
|
|
104
192
|
if (this.closed)
|
|
105
193
|
return;
|
|
106
194
|
this.closed = true;
|
|
107
|
-
this.
|
|
195
|
+
this.pending = [];
|
|
196
|
+
this.pendingBytes = 0;
|
|
108
197
|
this.results.clear();
|
|
109
198
|
if (!this.failed)
|
|
110
199
|
this.failWaiters(new CompletionFailedError('tool result stream closed'));
|
|
@@ -125,10 +214,11 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
|
|
|
125
214
|
// the turn that's currently generating — the expensive part a Stop button
|
|
126
215
|
// targets. Turns run strictly sequentially: turn N's request-registry entry
|
|
127
216
|
// is disposed (on the `for await` break below) before turn N+1 calls
|
|
128
|
-
// `begin(...)`, so reusing the id can't collide in `entries`.
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
217
|
+
// `begin(...)`, so reusing the id can't collide in `entries`. The handler
|
|
218
|
+
// registers each tool-result wait under the same id (see
|
|
219
|
+
// `registeredToolWaits`), so a cancel during the wait ends it; one that
|
|
220
|
+
// lands between registrations is caught by the registry's
|
|
221
|
+
// cancel-before-begin tripwire. Each turn's `history` and `type` are set below.
|
|
132
222
|
const { maxToolTurns, ...turnBase } = request;
|
|
133
223
|
const maxTurns = maxToolTurns ?? DEFAULT_MAX_TOOL_TURNS;
|
|
134
224
|
let history = [...request.history];
|
|
@@ -159,7 +249,7 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
|
|
|
159
249
|
const defers = request.tools?.some((tool) => tool.deferLoading === true) ?? false;
|
|
160
250
|
for (const call of toolCalls) {
|
|
161
251
|
// `tool_search` is the SDK's own tool: it reads the registered inventory
|
|
162
|
-
// and
|
|
252
|
+
// and records what it loaded in the history, so it never leaves the worker and
|
|
163
253
|
// the client is never asked to run it. The `toolCall` event still went
|
|
164
254
|
// downstream with the turn's events, so the client can see it happened.
|
|
165
255
|
if (defers && call.name === TOOL_SEARCH_NAME) {
|
|
@@ -167,7 +257,7 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
|
|
|
167
257
|
...history,
|
|
168
258
|
{
|
|
169
259
|
role: 'tool',
|
|
170
|
-
content: executeToolSearch(request
|
|
260
|
+
content: executeToolSearch(request, call.arguments, history)
|
|
171
261
|
}
|
|
172
262
|
];
|
|
173
263
|
continue;
|
|
@@ -177,7 +267,23 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
|
|
|
177
267
|
turn,
|
|
178
268
|
toolCallback: { callId: call.id, name: call.name, arguments: call.arguments }
|
|
179
269
|
};
|
|
180
|
-
|
|
270
|
+
let result;
|
|
271
|
+
try {
|
|
272
|
+
result = await reader.waitFor(call.id);
|
|
273
|
+
}
|
|
274
|
+
catch (error) {
|
|
275
|
+
if (!(error instanceof InferenceCancelledError))
|
|
276
|
+
throw error;
|
|
277
|
+
// Cancelled while the client ran its tool: end the way a cancelled
|
|
278
|
+
// turn ends, so the client folds a `cancelled` stop.
|
|
279
|
+
yield {
|
|
280
|
+
type: 'completionOrchestrate',
|
|
281
|
+
turn: turn + 1,
|
|
282
|
+
events: [{ type: 'completionDone', seq: 0, stopReason: 'cancelled' }]
|
|
283
|
+
};
|
|
284
|
+
yield { type: 'completionOrchestrate', done: true };
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
181
287
|
const content = result.error !== undefined
|
|
182
288
|
? JSON.stringify({ error: result.error })
|
|
183
289
|
: JSON.stringify(result.result ?? null);
|
|
@@ -190,13 +296,49 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
|
|
|
190
296
|
// whatever the model produced.
|
|
191
297
|
yield { type: 'completionOrchestrate', done: true, stopReason: 'maxToolTurns' };
|
|
192
298
|
}
|
|
299
|
+
/**
|
|
300
|
+
* Waits for a tool result as a request of its own, under the completion's
|
|
301
|
+
* `(sessionId, requestId)`, so the registry sees the completion between its
|
|
302
|
+
* turns: a cancel by id or by model, the transport signal and a session close
|
|
303
|
+
* stop the wait, and a session close waits for the completion to end.
|
|
304
|
+
*/
|
|
305
|
+
export function registeredToolWaits(reader, request) {
|
|
306
|
+
const requestId = request.requestId ?? generateRandomRequestId();
|
|
307
|
+
return {
|
|
308
|
+
async waitFor(callId) {
|
|
309
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
310
|
+
try {
|
|
311
|
+
// The model id lets `cancel({ modelId })` and an unload reach the wait.
|
|
312
|
+
// An unbounded cap skips slot admission, so the wait holds none of the
|
|
313
|
+
// model's completion slots while the client runs its tool.
|
|
314
|
+
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
315
|
+
requestId,
|
|
316
|
+
sessionId: request.sessionId,
|
|
317
|
+
kind: 'completion',
|
|
318
|
+
modelId: request.modelId,
|
|
319
|
+
maxConcurrentPerModel: Infinity
|
|
320
|
+
}), true);
|
|
321
|
+
return await reader.waitFor(callId, undefined, ctx.signal);
|
|
322
|
+
}
|
|
323
|
+
catch (e_1) {
|
|
324
|
+
env_1.error = e_1;
|
|
325
|
+
env_1.hasError = true;
|
|
326
|
+
}
|
|
327
|
+
finally {
|
|
328
|
+
const result_1 = __disposeResources(env_1);
|
|
329
|
+
if (result_1)
|
|
330
|
+
await result_1;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
};
|
|
334
|
+
}
|
|
193
335
|
export async function* handleCompletionOrchestrate(request, inputStream) {
|
|
194
336
|
const reader = new ToolResultReader(inputStream);
|
|
195
337
|
const runTurn = async function* (innerRequest) {
|
|
196
338
|
yield* dispatchPluginStream(innerRequest.modelId, 'completionStream', innerRequest);
|
|
197
339
|
};
|
|
198
340
|
try {
|
|
199
|
-
yield* orchestrateCompletion(request, runTurn, reader);
|
|
341
|
+
yield* orchestrateCompletion(request, runTurn, registeredToolWaits(reader, request));
|
|
200
342
|
}
|
|
201
343
|
finally {
|
|
202
344
|
// Tear the reader down however orchestration ends -- natural finish, turn
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { DeleteCacheRequest, DeleteCacheResponse } from '../schemas/index.js';
|
|
2
2
|
/**
|
|
3
|
-
* RPC handler for `deleteCache(...)`.
|
|
3
|
+
* RPC handler for `deleteCache(...)`. `modelSrc` deletes a registry model's
|
|
4
|
+
* cached download instead of KV caches. The three KV-cache bookkeeping
|
|
4
5
|
* layers (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`)
|
|
5
6
|
* are private to `kv-cache-session.ts`; this handler delegates to that
|
|
6
7
|
* module's single administrative entry point (`deleteKvCacheState`).
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { deleteKvCacheState } from '../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
|
|
2
|
+
import { deleteRegistryModelCache } from '../handlers/load-model/registry-delete.js';
|
|
2
3
|
import { getEngineLogger } from '../logging/index.js';
|
|
3
4
|
const logger = getEngineLogger();
|
|
4
5
|
/**
|
|
5
|
-
* RPC handler for `deleteCache(...)`.
|
|
6
|
+
* RPC handler for `deleteCache(...)`. `modelSrc` deletes a registry model's
|
|
7
|
+
* cached download instead of KV caches. The three KV-cache bookkeeping
|
|
6
8
|
* layers (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`)
|
|
7
9
|
* are private to `kv-cache-session.ts`; this handler delegates to that
|
|
8
10
|
* module's single administrative entry point (`deleteKvCacheState`).
|
|
@@ -18,6 +20,9 @@ export async function handleDeleteCache(request) {
|
|
|
18
20
|
else if ('auto' in request && request.auto) {
|
|
19
21
|
await deleteKvCacheState({ auto: true });
|
|
20
22
|
}
|
|
23
|
+
else if ('modelSrc' in request) {
|
|
24
|
+
await deleteRegistryModelCache(request.modelSrc);
|
|
25
|
+
}
|
|
21
26
|
else if ('kvCacheKey' in request) {
|
|
22
27
|
await deleteKvCacheState({
|
|
23
28
|
kvCacheKey: request.kvCacheKey,
|
|
@@ -51,13 +51,14 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
53
|
import { PROFILING_KEY, OPERATION_EVENT_KEY } from '../schemas/index.js';
|
|
54
|
-
import { resolveModelPath, resolveModelPathWithStats } from '
|
|
55
|
-
import { buildDownloadProfilingFields } from '
|
|
54
|
+
import { resolveModelPath, resolveModelPathWithStats } from '../handlers/load-model/resolve.js';
|
|
55
|
+
import { buildDownloadProfilingFields } from '../handlers/load-model/types.js';
|
|
56
56
|
import { nowMs, generateProfileId } from '../profiling/clock.js';
|
|
57
57
|
import { getEngineLogger } from '../logging/index.js';
|
|
58
58
|
import { getRequestRegistry, withRequestContext } from '../runtime/index.js';
|
|
59
59
|
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
60
|
-
import {
|
|
60
|
+
import { resolveSessionId } from '../runtime/session-id.js';
|
|
61
|
+
import { InferenceCancelledError, ModelRegistryBusyError } from '../errors/index.js';
|
|
61
62
|
const logger = getEngineLogger();
|
|
62
63
|
export async function handleDownloadAsset(request, progressCallback) {
|
|
63
64
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
@@ -67,12 +68,14 @@ export async function handleDownloadAsset(request, progressCallback) {
|
|
|
67
68
|
const profilingMeta = request[PROFILING_KEY];
|
|
68
69
|
const profilingEnabled = profilingMeta?.enabled !== false && !!profilingMeta;
|
|
69
70
|
const requestId = request.requestId ?? generateRandomRequestId();
|
|
71
|
+
const sessionId = resolveSessionId(request);
|
|
70
72
|
// `downloadAsset` is artifact-shaped, not model-shaped — there is no
|
|
71
73
|
// `modelId` to register on the registry entry. Cancel by `requestId`
|
|
72
74
|
// is the primary path; `cancel({ modelId })` is intentionally a
|
|
73
75
|
// non-match for this kind.
|
|
74
76
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
75
77
|
requestId,
|
|
78
|
+
sessionId,
|
|
76
79
|
kind: 'downloadAsset'
|
|
77
80
|
}), true);
|
|
78
81
|
const log = withRequestContext(getEngineLogger(), ctx);
|
|
@@ -81,7 +84,8 @@ export async function handleDownloadAsset(request, progressCallback) {
|
|
|
81
84
|
requestBinding: {
|
|
82
85
|
signal: ctx.signal,
|
|
83
86
|
scope: ctx.scope,
|
|
84
|
-
requestId
|
|
87
|
+
requestId,
|
|
88
|
+
sessionId
|
|
85
89
|
}
|
|
86
90
|
};
|
|
87
91
|
try {
|
|
@@ -121,12 +125,15 @@ export async function handleDownloadAsset(request, progressCallback) {
|
|
|
121
125
|
catch (error) {
|
|
122
126
|
// Mirror the load-model handler's cancel contract: a typed cancel
|
|
123
127
|
// bubbles up as the rejection so client-side callers can branch on
|
|
124
|
-
// `instanceof InferenceCancelledError
|
|
125
|
-
// the legacy `success: false` envelope.
|
|
128
|
+
// `instanceof InferenceCancelledError`, and so does a busy registry.
|
|
129
|
+
// Every other error keeps the legacy `success: false` envelope.
|
|
126
130
|
if (error instanceof InferenceCancelledError) {
|
|
127
131
|
log.info(`downloadAsset cancelled requestId=${requestId}`);
|
|
128
132
|
throw error;
|
|
129
133
|
}
|
|
134
|
+
// Typed so callers can tell another process has the registry.
|
|
135
|
+
if (error instanceof ModelRegistryBusyError)
|
|
136
|
+
throw error;
|
|
130
137
|
logger.error('Error downloading asset:', error);
|
|
131
138
|
return {
|
|
132
139
|
type: 'downloadAsset',
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { dispatchPluginReply } from '
|
|
1
|
+
import { dispatchPluginReply } from '../handlers/plugin-dispatch.js';
|
|
2
2
|
import { startFinetune } from '../plugins/builtin/llamacpp-completion/ops/finetune.js';
|
|
3
3
|
export async function handleFinetune(request, progressCallback) {
|
|
4
4
|
if (progressCallback &&
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { getModelEntry } from '../runtime/model-registry.js';
|
|
2
2
|
import { getPlugin } from '../plugins/registry.js';
|
|
3
|
-
import {
|
|
3
|
+
import { detectToolDialect } from '../utils/tool-integration.js';
|
|
4
4
|
import { ModelNotFoundError } from '../errors/index.js';
|
|
5
5
|
import { getEngineLogger } from '../logging/index.js';
|
|
6
6
|
const logger = getEngineLogger();
|
|
@@ -21,13 +21,13 @@ export function handleGetLoadedModelInfo(request) {
|
|
|
21
21
|
modelType: entry.local.modelType,
|
|
22
22
|
handlers,
|
|
23
23
|
loadedAt: entry.local.loadedAt,
|
|
24
|
+
leaseCount: entry.leases.size,
|
|
24
25
|
...(plugin && { displayName: plugin.displayName }),
|
|
25
26
|
...(plugin && { addonPackage: plugin.addonPackage }),
|
|
26
27
|
...(entry.local.name && { name: entry.local.name }),
|
|
27
28
|
...(entry.local.path && { path: entry.local.path }),
|
|
28
|
-
// Same detection the completion normalizer uses, so callers see the dialect it parses.
|
|
29
29
|
...(handlers.includes('completionStream') && {
|
|
30
|
-
toolDialect:
|
|
30
|
+
toolDialect: detectToolDialect(modelId)
|
|
31
31
|
}),
|
|
32
32
|
...(fitProbe !== undefined && { fitProbe })
|
|
33
33
|
};
|
|
@@ -1,3 +1,55 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
1
53
|
import { models } from '../models/registry/models.js';
|
|
2
54
|
import { getAllModelIds, getModelEntry } from '../runtime/model-registry.js';
|
|
3
55
|
import { generateShortHash } from '../utils/index.js';
|
|
@@ -5,8 +57,8 @@ import { promises as fsPromises } from 'bare-fs';
|
|
|
5
57
|
import { getShardPath, getModelsCacheDir, getSingleFileCachePath } from '../utils/cache/paths.js';
|
|
6
58
|
import { validateAndJoinPath } from '../utils/path-security.js';
|
|
7
59
|
import { ModelNotFoundError } from '../errors/index.js';
|
|
8
|
-
import { findCoremlCompanionSet, findLocallyCachedCoremlCompanionSet, getCoremlSidecarSpec, getCoremlSetKey } from '
|
|
9
|
-
import {
|
|
60
|
+
import { findCoremlCompanionSet, findLocallyCachedCoremlCompanionSet, getCoremlSidecarSpec, getCoremlSetKey } from '../handlers/load-model/coreml-sidecars.js';
|
|
61
|
+
import { acquireRegistryClient } from '../runtime/registry-client.js';
|
|
10
62
|
import { getRuntimeContext } from '../runtime/state.js';
|
|
11
63
|
export async function handleGetModelInfo(request) {
|
|
12
64
|
const { name } = request;
|
|
@@ -81,7 +133,24 @@ async function handleOptionalCoremlModel(model) {
|
|
|
81
133
|
const filename = model.registryPath.split('/').pop() || model.registryPath;
|
|
82
134
|
const primaryPath = validateAndJoinPath(getModelsCacheDir(), 'sets', getCoremlSetKey(model, spec), filename);
|
|
83
135
|
await fsPromises.access(primaryPath);
|
|
84
|
-
|
|
136
|
+
let set;
|
|
137
|
+
{
|
|
138
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
139
|
+
try {
|
|
140
|
+
// A status check reports what is on disk rather than wait on another process.
|
|
141
|
+
const lease = __addDisposableResource(env_1, await acquireRegistryClient({ wait: 'status' }), true);
|
|
142
|
+
set = await findCoremlCompanionSet(lease.client, model, platform);
|
|
143
|
+
}
|
|
144
|
+
catch (e_1) {
|
|
145
|
+
env_1.error = e_1;
|
|
146
|
+
env_1.hasError = true;
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
const result_1 = __disposeResources(env_1);
|
|
150
|
+
if (result_1)
|
|
151
|
+
await result_1;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
85
154
|
if (set) {
|
|
86
155
|
const status = await handleCompanionSetModel(set, model.registryPath, model.addon);
|
|
87
156
|
if (status.isCached)
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
import type { QVACRegistryClient } from '@qvac/registry-client';
|
|
2
2
|
import type { RegistryItem } from '../../models/registry/models.js';
|
|
3
|
-
type
|
|
4
|
-
|
|
5
|
-
targetSuffix: string;
|
|
6
|
-
};
|
|
7
|
-
export type CoremlSidecarSpec = {
|
|
8
|
-
root: string;
|
|
9
|
-
modelStem: string;
|
|
10
|
-
bundles: readonly BundleSpec[];
|
|
11
|
-
};
|
|
3
|
+
import { getCoremlSidecarSpec, type CoremlSidecarSpec } from '../../utils/cache/layout.js';
|
|
4
|
+
export { getCoremlSidecarSpec, type CoremlSidecarSpec };
|
|
12
5
|
type CompanionSet = NonNullable<RegistryItem['companionSet']>;
|
|
13
|
-
|
|
6
|
+
/**
|
|
7
|
+
* For the encoder-less GGUF of a Core ML distribution, the model constant to
|
|
8
|
+
* load instead where its bundles cannot be staged. The loader stages bundles
|
|
9
|
+
* on macOS and iOS alone, so anywhere else such a load is refused up front
|
|
10
|
+
* rather than failing inside the addon after the download.
|
|
11
|
+
*/
|
|
12
|
+
export declare function getCoremlOnlyAlternative(registryPath: string, registrySource: string): string | undefined;
|
|
13
|
+
/**
|
|
14
|
+
* Why a `registry://` model source cannot load on `platform`, or `undefined`
|
|
15
|
+
* when it can. Only the encoder-less GGUF of a Core ML distribution is refused,
|
|
16
|
+
* and only off Apple platforms; every other source is the loader's business.
|
|
17
|
+
*/
|
|
18
|
+
export declare function getRegistryModelPlatformRefusal(modelSrc: unknown, platform: string | undefined): string | undefined;
|
|
14
19
|
export declare function getCoremlSetKey(model: RegistryItem, spec: CoremlSidecarSpec): string;
|
|
15
20
|
export declare function readCachedCoremlCompanionSet(model: RegistryItem, platform: string | undefined, cacheDir?: string): Promise<CompanionSet | undefined>;
|
|
16
21
|
export declare function writeCachedCoremlCompanionSet(set: CompanionSet, cacheDir?: string): Promise<void>;
|
|
@@ -22,4 +27,3 @@ export declare function findLocallyCachedCoremlCompanionSet(model: RegistryItem,
|
|
|
22
27
|
* this lets the SDK use them as soon as the registry has ingested the files.
|
|
23
28
|
*/
|
|
24
29
|
export declare function findCoremlCompanionSet(client: Pick<QVACRegistryClient, 'findModels'>, model: RegistryItem, platform: string | undefined): Promise<CompanionSet | undefined>;
|
|
25
|
-
export {};
|