@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -1,307 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { inferModelTypeFromModelSrc } from '../schemas/model-src-utils.js';
|
|
3
|
-
import { ModelTypeRequiredError } from '../errors/index.js';
|
|
4
|
-
import { projectFitFromLoad } from '../model-fit/description/project.js';
|
|
5
|
-
import { getResourceCollector } from '../resources/instance.js';
|
|
6
|
-
import { getConfig } from '../runtime/state.js';
|
|
7
|
-
import { assessModelFitFromResources } from '../model-fit/assess.js';
|
|
8
|
-
import { detectPlatform } from '../model-fit/platform.js';
|
|
1
|
+
import { assessModelFit } from '../model-fit/assess.js';
|
|
9
2
|
/**
|
|
10
|
-
* Runs a
|
|
11
|
-
*
|
|
12
|
-
* This lives on the worker because that is where the three things it needs
|
|
13
|
-
* already are: the resource collector for a fresh memory sample, the runtime's
|
|
14
|
-
* own platform/arch pair, and the registry client. No weights are read and
|
|
15
|
-
* nothing is loaded — each candidate additionally has the registry's weightless
|
|
16
|
-
* description fetched, tens of KB, so the engine's own fitter can answer.
|
|
3
|
+
* Runs a fit assessment worker-side, where the plugins, the registry client
|
|
4
|
+
* and the memory sample are. No weights are read and nothing is loaded.
|
|
17
5
|
*/
|
|
18
6
|
export async function handleAssessModelFit(request) {
|
|
19
|
-
|
|
20
|
-
const nativeFits = await resolveNativeFits(request.models);
|
|
21
|
-
const result = assessModelFitFromResources({
|
|
22
|
-
models: request.models.map(estimateTargetFor),
|
|
23
|
-
execution: request.execution,
|
|
24
|
-
resources: readResources(),
|
|
25
|
-
platform,
|
|
26
|
-
nativeFits
|
|
27
|
-
});
|
|
28
|
-
return { type: 'assessModelFit', ...result };
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* The engine that would run this load, named outright or read off the source,
|
|
32
|
-
* as `loadModel` resolves it.
|
|
33
|
-
*
|
|
34
|
-
* @throws {ModelTypeRequiredError} When neither names one.
|
|
35
|
-
*/
|
|
36
|
-
function modelTypeOf(candidate) {
|
|
37
|
-
if (candidate.modelType !== undefined)
|
|
38
|
-
return normalizeModelType(candidate.modelType);
|
|
39
|
-
const inferred = inferModelTypeFromModelSrc(candidate.modelSrc);
|
|
40
|
-
if (inferred === undefined)
|
|
41
|
-
throw new ModelTypeRequiredError();
|
|
42
|
-
return normalizeModelType(inferred);
|
|
43
|
-
}
|
|
44
|
-
/** The audio engines, whose estimator sizes a load by its window rather than a context. */
|
|
45
|
-
const AUDIO_ENGINES = [
|
|
46
|
-
ModelType.mossTranscribe,
|
|
47
|
-
'whispercpp-transcription',
|
|
48
|
-
'parakeet-transcription',
|
|
49
|
-
'bci-whispercpp-transcription'
|
|
50
|
-
];
|
|
51
|
-
/**
|
|
52
|
-
* Engines that name a device outright, in llama's own `gpu` / `cpu` spelling.
|
|
53
|
-
* Both default to the GPU.
|
|
54
|
-
*/
|
|
55
|
-
const DEVICE_NAMED = [ModelType.llamacppCompletion, ModelType.llamacppEmbedding];
|
|
56
|
-
function contextUsesGpu(config) {
|
|
57
|
-
const contextParams = config['contextParams'];
|
|
58
|
-
if (contextParams === null || typeof contextParams !== 'object')
|
|
59
|
-
return undefined;
|
|
60
|
-
return contextParams['use_gpu'];
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* MiniMax names its device instead of switching the GPU on, and the config
|
|
64
|
-
* schema refuses both at once. `auto` is what the switch means: the GPU when
|
|
65
|
-
* one is usable.
|
|
66
|
-
*/
|
|
67
|
-
function audiogenUsesGpu(config) {
|
|
68
|
-
const device = config['device'];
|
|
69
|
-
if (device !== undefined)
|
|
70
|
-
return device === 'gpu' || device === 'auto';
|
|
71
|
-
return config['useGPU'] === true;
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Engines that carry a GPU switch, each read at the key its own config spells
|
|
75
|
-
* it under. These are the keys the fit builders read in `model-fit/request/engines`,
|
|
76
|
-
* so a projection and a device never disagree. Only bci defaults to on.
|
|
77
|
-
*/
|
|
78
|
-
const GPU_SWITCH = {
|
|
79
|
-
[ModelType.mossTranscribe]: (config) => config['useGPU'] === true,
|
|
80
|
-
[ModelType.ttsGgml]: (config) => config['useGPU'] === true,
|
|
81
|
-
[ModelType.audiogenGgml]: audiogenUsesGpu,
|
|
82
|
-
[ModelType.parakeetTranscription]: (config) => config['useGPU'] === true,
|
|
83
|
-
[ModelType.whispercppTranscription]: (config) => contextUsesGpu(config) === true,
|
|
84
|
-
[ModelType.bciWhispercppTranscription]: (config) => contextUsesGpu(config) !== false,
|
|
85
|
-
[ModelType.ggmlOcr]: (config) => (config['backendDevice'] ?? 'cpu') !== 'cpu'
|
|
86
|
-
};
|
|
87
|
-
/**
|
|
88
|
-
* Where a load resolved to run. `dispatch` applies the host's device defaults
|
|
89
|
-
* before any handler sees the config, so what arrives here is the resolved one.
|
|
90
|
-
*
|
|
91
|
-
* `nGpuLayers` wins over `useGPU` on tts, and the config schema rejects the two
|
|
92
|
-
* disagreeing, so reading it first is safe.
|
|
93
|
-
*/
|
|
94
|
-
function resolvedDevice(modelType, config) {
|
|
95
|
-
if (DEVICE_NAMED.includes(modelType)) {
|
|
96
|
-
const device = config['device'];
|
|
97
|
-
return typeof device === 'string' ? device.toLowerCase() : 'gpu';
|
|
98
|
-
}
|
|
99
|
-
const layers = config['nGpuLayers'];
|
|
100
|
-
if (modelType === ModelType.ttsGgml && typeof layers === 'number') {
|
|
101
|
-
return layers === 0 ? 'cpu' : 'gpu';
|
|
102
|
-
}
|
|
103
|
-
const usesGpu = GPU_SWITCH[modelType];
|
|
104
|
-
if (usesGpu === undefined)
|
|
105
|
-
return undefined;
|
|
106
|
-
return usesGpu(config) ? 'gpu' : 'cpu';
|
|
107
|
-
}
|
|
108
|
-
/** The window the speech engines hold whole; longer audio is chunked into it. */
|
|
109
|
-
const AUDIO_WINDOW_MS = 30_000;
|
|
110
|
-
/**
|
|
111
|
-
* The audio a single call holds in memory. `duration_ms` is the longest clip
|
|
112
|
-
* the caller will transcribe, not a per-call window, and the engine chunks
|
|
113
|
-
* anything longer, so it never sizes working memory above one window.
|
|
114
|
-
*/
|
|
115
|
-
function audioWindowMs(config) {
|
|
116
|
-
const declared = config['duration_ms'];
|
|
117
|
-
if (typeof declared !== 'number' || declared <= 0)
|
|
118
|
-
return AUDIO_WINDOW_MS;
|
|
119
|
-
return Math.min(declared, AUDIO_WINDOW_MS);
|
|
120
|
-
}
|
|
121
|
-
/** The catalog facts a source carries, where it carries them. */
|
|
122
|
-
function modelRefOf(value) {
|
|
123
|
-
if (value === null || typeof value !== 'object')
|
|
124
|
-
return undefined;
|
|
125
|
-
const src = value;
|
|
126
|
-
if (typeof src['sha256Checksum'] !== 'string')
|
|
127
|
-
return undefined;
|
|
128
|
-
const name = src['name'] ?? src['modelId'] ?? src['registryPath'];
|
|
129
|
-
return {
|
|
130
|
-
name: typeof name === 'string' ? name : '',
|
|
131
|
-
sha256Checksum: src['sha256Checksum'],
|
|
132
|
-
...(typeof src['registryPath'] === 'string' && { registryPath: src['registryPath'] }),
|
|
133
|
-
...(typeof src['registrySource'] === 'string' && { registrySource: src['registrySource'] })
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
/**
|
|
137
|
-
* The companion sources a load carries in its config — a VAD model, a pivot,
|
|
138
|
-
* an audiogen stage, a projector. Their bytes are part of what the set costs,
|
|
139
|
-
* so the estimate counts them alongside the primary.
|
|
140
|
-
*/
|
|
141
|
-
function companionRefs(config) {
|
|
142
|
-
const refs = [];
|
|
143
|
-
const seen = new Set();
|
|
144
|
-
const add = (ref) => {
|
|
145
|
-
if (seen.has(ref.sha256Checksum))
|
|
146
|
-
return;
|
|
147
|
-
seen.add(ref.sha256Checksum);
|
|
148
|
-
refs.push(ref);
|
|
149
|
-
};
|
|
150
|
-
const walk = (key, value) => {
|
|
151
|
-
const ref = modelRefOf(value);
|
|
152
|
-
if (ref !== undefined) {
|
|
153
|
-
add(ref);
|
|
154
|
-
return;
|
|
155
|
-
}
|
|
156
|
-
if (key.endsWith('ModelSrc')) {
|
|
157
|
-
add({ name: typeof value === 'string' ? value : key, sha256Checksum: '' });
|
|
158
|
-
return;
|
|
159
|
-
}
|
|
160
|
-
if (value === null || typeof value !== 'object')
|
|
161
|
-
return;
|
|
162
|
-
for (const [inner, nested] of Object.entries(value)) {
|
|
163
|
-
walk(inner, nested);
|
|
164
|
-
}
|
|
165
|
-
};
|
|
166
|
-
for (const [key, value] of Object.entries(config))
|
|
167
|
-
walk(key, value);
|
|
168
|
-
return refs;
|
|
169
|
-
}
|
|
170
|
-
/**
|
|
171
|
-
* What the estimator is given for one candidate. The catalog facts come from
|
|
172
|
-
* the source's checksum, and the workload from the settings the load carries;
|
|
173
|
-
* a source outside the catalog resolves neither and assesses as `unknown`.
|
|
174
|
-
*
|
|
175
|
-
* A load whose sources are all config fields carries no checksum, so it has no
|
|
176
|
-
* catalog profile and the engine's own fitter is its only evidence. The model
|
|
177
|
-
* type stands in as the label.
|
|
178
|
-
*/
|
|
179
|
-
export function estimateTargetFor(candidate) {
|
|
180
|
-
const descriptor = typeof candidate.modelSrc === 'string' ? undefined : candidate.modelSrc;
|
|
181
|
-
const location = typeof candidate.modelSrc === 'string' ? candidate.modelSrc : undefined;
|
|
182
|
-
const config = candidate.modelConfig ?? {};
|
|
183
|
-
const modelType = modelTypeOf(candidate);
|
|
184
|
-
const contextTokens = config['ctx_size'];
|
|
185
|
-
const artifacts = companionRefs(config);
|
|
186
|
-
const device = resolvedDevice(modelType, config);
|
|
187
|
-
const windowMs = modelType === ModelType.mossTranscribe
|
|
188
|
-
? candidate.transcriptionWorkload === undefined
|
|
189
|
-
? undefined
|
|
190
|
-
: candidate.transcriptionWorkload.audioSeconds * 1000
|
|
191
|
-
: audioWindowMs(config);
|
|
192
|
-
return {
|
|
193
|
-
model: {
|
|
194
|
-
name: descriptor?.name ?? descriptor?.modelId ?? location ?? modelType,
|
|
195
|
-
sha256Checksum: descriptor?.sha256Checksum ?? '',
|
|
196
|
-
...(descriptor?.registryPath !== undefined && { registryPath: descriptor.registryPath }),
|
|
197
|
-
...(descriptor?.registrySource !== undefined && {
|
|
198
|
-
registrySource: descriptor.registrySource
|
|
199
|
-
})
|
|
200
|
-
},
|
|
201
|
-
...(artifacts.length > 0 && { artifacts }),
|
|
202
|
-
...(device !== undefined && { device }),
|
|
203
|
-
workload: AUDIO_ENGINES.includes(modelType)
|
|
204
|
-
? {
|
|
205
|
-
kind: 'audio',
|
|
206
|
-
...(windowMs !== undefined && { windowMs }),
|
|
207
|
-
streaming: config['streaming'] === true
|
|
208
|
-
}
|
|
209
|
-
: {
|
|
210
|
-
kind: 'llm',
|
|
211
|
-
...(typeof contextTokens === 'number' && contextTokens > 0 && { contextTokens })
|
|
212
|
-
}
|
|
213
|
-
};
|
|
214
|
-
}
|
|
215
|
-
/** The engine fitter's verdict for one candidate, through that load's plugin. */
|
|
216
|
-
async function resolveNativeFit(candidate, alreadyCounted) {
|
|
217
|
-
const modelType = modelTypeOf(candidate);
|
|
218
|
-
if (!isCanonicalModelType(modelType)) {
|
|
219
|
-
return { unavailable: `no plugin handles model type ${modelType}` };
|
|
220
|
-
}
|
|
221
|
-
const budgetMs = getConfig().fitStubBudgetMs;
|
|
222
|
-
const outcome = await projectFitFromLoad({
|
|
223
|
-
modelType,
|
|
224
|
-
...(candidate.transcriptionWorkload !== undefined && {
|
|
225
|
-
transcriptionWorkload: candidate.transcriptionWorkload
|
|
226
|
-
}),
|
|
227
|
-
...(candidate.ttsWorkload !== undefined && { ttsWorkload: candidate.ttsWorkload }),
|
|
228
|
-
...(candidate.modelSrc !== undefined && { modelSrc: candidate.modelSrc }),
|
|
229
|
-
...(candidate.modelConfig !== undefined && { modelConfig: candidate.modelConfig })
|
|
230
|
-
}, estimateTargetFor(candidate).model.name, {
|
|
231
|
-
...(budgetMs !== undefined && { stub: { budgetMs } }),
|
|
232
|
-
...(alreadyCounted > 0 && { fit: { extraResidentBytes: alreadyCounted } })
|
|
233
|
-
});
|
|
234
|
-
if (outcome.status === 'projected')
|
|
235
|
-
return { fit: outcome.fit };
|
|
236
|
-
if (outcome.status === 'unsupported-load')
|
|
237
|
-
return { unavailable: outcome.detail };
|
|
238
|
-
return { unavailable: noStubDetail(outcome.reason, outcome.message) };
|
|
239
|
-
}
|
|
240
|
-
/** Why a load with no description staged for it has no engine verdict. */
|
|
241
|
-
export function noStubDetail(reason, message) {
|
|
242
|
-
if (reason === 'files-only-engine') {
|
|
243
|
-
return 'the engine measures downloaded files only, and this model is not downloaded';
|
|
244
|
-
}
|
|
245
|
-
return message === undefined
|
|
246
|
-
? `no registry description (${reason})`
|
|
247
|
-
: `no registry description (${reason}): ${message}`;
|
|
248
|
-
}
|
|
249
|
-
/** What a projection holds for the model's lifetime, the breakdown or the total. */
|
|
250
|
-
function residentBytesOf(fit) {
|
|
251
|
-
const projection = fit.fit?.projection;
|
|
252
|
-
if (!projection)
|
|
253
|
-
return 0;
|
|
254
|
-
const { weightsBytes, contextBytes, deviceBytes, hostBytes } = projection;
|
|
255
|
-
const resident = weightsBytes === undefined || contextBytes === undefined
|
|
256
|
-
? deviceBytes
|
|
257
|
-
: weightsBytes + contextBytes;
|
|
258
|
-
return (resident ?? 0) + (hostBytes ?? 0);
|
|
259
|
-
}
|
|
260
|
-
/**
|
|
261
|
-
* One verdict per candidate, in request order. Each probe measures its own
|
|
262
|
-
* model against the whole machine, so the bytes compose where the verdicts do
|
|
263
|
-
* not, and `assess` combines them under one budget.
|
|
264
|
-
*
|
|
265
|
-
* Run one at a time, so a set never holds several fitters open at once.
|
|
266
|
-
*
|
|
267
|
-
* Each probe holds back what the earlier ones projected, since none of them is
|
|
268
|
-
* registered in this worker and the fitter would otherwise place every
|
|
269
|
-
* candidate in the same free memory. Every model is resident under either
|
|
270
|
-
* execution mode, so the holdback does not read it. The answer follows the
|
|
271
|
-
* order the caller listed the models in.
|
|
272
|
-
*/
|
|
273
|
-
async function resolveNativeFits(candidates) {
|
|
274
|
-
const fits = [];
|
|
275
|
-
let alreadyCounted = 0;
|
|
276
|
-
for (const candidate of candidates) {
|
|
277
|
-
const fit = await resolveNativeFit(candidate, alreadyCounted);
|
|
278
|
-
fits.push(fit);
|
|
279
|
-
alreadyCounted += residentBytesOf(fit);
|
|
280
|
-
}
|
|
281
|
-
return fits;
|
|
282
|
-
}
|
|
283
|
-
function readResources() {
|
|
284
|
-
const collector = getResourceCollector();
|
|
285
|
-
if (!collector) {
|
|
286
|
-
const failed = { status: 'failed', reason: 'resource collector is not initialized' };
|
|
287
|
-
return {
|
|
288
|
-
capabilities: {
|
|
289
|
-
cpu: failed,
|
|
290
|
-
memory: { totalBytes: failed },
|
|
291
|
-
gpus: failed
|
|
292
|
-
},
|
|
293
|
-
sample: {
|
|
294
|
-
sampledAt: Date.now(),
|
|
295
|
-
cpu: failed,
|
|
296
|
-
memory: {
|
|
297
|
-
usedBytes: failed,
|
|
298
|
-
totalBytes: failed,
|
|
299
|
-
processUsedBytes: failed,
|
|
300
|
-
processAvailableBytes: failed
|
|
301
|
-
},
|
|
302
|
-
gpus: failed
|
|
303
|
-
}
|
|
304
|
-
};
|
|
305
|
-
}
|
|
306
|
-
return { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
7
|
+
return { type: 'assessModelFit', ...(await assessModelFit(request.models)) };
|
|
307
8
|
}
|
|
@@ -5,16 +5,16 @@ import type { CancelRequest, CancelResponse } from '../schemas/cancel.js';
|
|
|
5
5
|
* surface is two paths that route through the same registry primitive:
|
|
6
6
|
*
|
|
7
7
|
* - `{ operation: "request", requestId, clearCache? }` — targeted
|
|
8
|
-
* cancel by caller-generated id
|
|
9
|
-
* fires its abort signal,
|
|
10
|
-
* transfer for cache clear.
|
|
11
|
-
* beats its own begin) is
|
|
12
|
-
* registry via the cancel-before-begin tripwire.
|
|
8
|
+
* cancel by caller-generated id, resolved within the caller's own
|
|
9
|
+
* session. Looks up the registry entry, fires its abort signal,
|
|
10
|
+
* optionally marks the underlying download transfer for cache clear.
|
|
11
|
+
* The "stop-button race" case (a cancel beats its own begin) is
|
|
12
|
+
* handled inside the registry via the cancel-before-begin tripwire.
|
|
13
13
|
*
|
|
14
14
|
* - `{ operation: "broad", modelId, kind? }` — abort every in-flight
|
|
15
|
-
* request on a model (optionally narrowed
|
|
16
|
-
*
|
|
17
|
-
*
|
|
15
|
+
* request the caller's session has on a model (optionally narrowed
|
|
16
|
+
* by `kind`). From the owner session it reaches every session's
|
|
17
|
+
* requests on the model. Delegates to the `cancel` bare op so the
|
|
18
18
|
* `ModelNotLoadedError` validation is shared with internal
|
|
19
19
|
* engine-side broad cancels.
|
|
20
20
|
*
|
|
@@ -24,6 +24,7 @@ import type { CancelRequest, CancelResponse } from '../schemas/cancel.js';
|
|
|
24
24
|
* match still returns `success: true, cancelled: 0`; the
|
|
25
25
|
* cancel-before-begin tripwire ensures the cancel is applied
|
|
26
26
|
* retroactively if a matching begin arrives within the registry's
|
|
27
|
-
* race window.
|
|
27
|
+
* race window. Another session's request id is such a non-match: the
|
|
28
|
+
* registry keys requests by `(sessionId, requestId)`, so it is never found.
|
|
28
29
|
*/
|
|
29
30
|
export declare function cancelHandler(request: CancelRequest): CancelResponse;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { cancel as cancelByModelId } from '../plugins/ops/cancel.js';
|
|
2
2
|
import { getRequestRegistry } from '../runtime/index.js';
|
|
3
|
+
import { resolveSessionId } from '../runtime/session-id.js';
|
|
3
4
|
import { markClearCacheForRequest } from '../handlers/load-model/download-manager.js';
|
|
4
5
|
import { getEngineLogger } from '../logging/index.js';
|
|
5
6
|
const logger = getEngineLogger();
|
|
@@ -9,16 +10,16 @@ const logger = getEngineLogger();
|
|
|
9
10
|
* surface is two paths that route through the same registry primitive:
|
|
10
11
|
*
|
|
11
12
|
* - `{ operation: "request", requestId, clearCache? }` — targeted
|
|
12
|
-
* cancel by caller-generated id
|
|
13
|
-
* fires its abort signal,
|
|
14
|
-
* transfer for cache clear.
|
|
15
|
-
* beats its own begin) is
|
|
16
|
-
* registry via the cancel-before-begin tripwire.
|
|
13
|
+
* cancel by caller-generated id, resolved within the caller's own
|
|
14
|
+
* session. Looks up the registry entry, fires its abort signal,
|
|
15
|
+
* optionally marks the underlying download transfer for cache clear.
|
|
16
|
+
* The "stop-button race" case (a cancel beats its own begin) is
|
|
17
|
+
* handled inside the registry via the cancel-before-begin tripwire.
|
|
17
18
|
*
|
|
18
19
|
* - `{ operation: "broad", modelId, kind? }` — abort every in-flight
|
|
19
|
-
* request on a model (optionally narrowed
|
|
20
|
-
*
|
|
21
|
-
*
|
|
20
|
+
* request the caller's session has on a model (optionally narrowed
|
|
21
|
+
* by `kind`). From the owner session it reaches every session's
|
|
22
|
+
* requests on the model. Delegates to the `cancel` bare op so the
|
|
22
23
|
* `ModelNotLoadedError` validation is shared with internal
|
|
23
24
|
* engine-side broad cancels.
|
|
24
25
|
*
|
|
@@ -28,16 +29,19 @@ const logger = getEngineLogger();
|
|
|
28
29
|
* match still returns `success: true, cancelled: 0`; the
|
|
29
30
|
* cancel-before-begin tripwire ensures the cancel is applied
|
|
30
31
|
* retroactively if a matching begin arrives within the registry's
|
|
31
|
-
* race window.
|
|
32
|
+
* race window. Another session's request id is such a non-match: the
|
|
33
|
+
* registry keys requests by `(sessionId, requestId)`, so it is never found.
|
|
32
34
|
*/
|
|
33
35
|
export function cancelHandler(request) {
|
|
36
|
+
const sessionId = resolveSessionId(request);
|
|
34
37
|
try {
|
|
35
38
|
if (request.operation === 'request') {
|
|
36
39
|
if (request.clearCache) {
|
|
37
|
-
markClearCacheForRequest(request.requestId);
|
|
40
|
+
markClearCacheForRequest(request.requestId, sessionId);
|
|
38
41
|
}
|
|
39
42
|
const cancelled = getRequestRegistry().cancel({
|
|
40
|
-
requestId: request.requestId
|
|
43
|
+
requestId: request.requestId,
|
|
44
|
+
sessionId
|
|
41
45
|
});
|
|
42
46
|
if (cancelled === 0) {
|
|
43
47
|
// info-level (not debug) because the decorated-promise pattern
|
|
@@ -48,12 +52,12 @@ export function cancelHandler(request) {
|
|
|
48
52
|
// matching begin in flight; this log just helps operators
|
|
49
53
|
// debugging "my Stop button isn't working" without lowering
|
|
50
54
|
// the log level.
|
|
51
|
-
logger.info(`[cancel] no in-flight request matched requestId=${request.requestId}`);
|
|
55
|
+
logger.info(`[cancel] no in-flight request matched sessionId=${sessionId} requestId=${request.requestId}`);
|
|
52
56
|
}
|
|
53
57
|
return { type: 'cancel', success: true, cancelled };
|
|
54
58
|
}
|
|
55
59
|
// operation === "broad"
|
|
56
|
-
const cancelled = cancelByModelId({ modelId: request.modelId }, request.kind
|
|
60
|
+
const cancelled = cancelByModelId({ modelId: request.modelId }, { ...(request.kind && { kind: request.kind }), sessionId });
|
|
57
61
|
return { type: 'cancel', success: true, cancelled };
|
|
58
62
|
}
|
|
59
63
|
catch (error) {
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import Buffer from 'bare-buffer';
|
|
1
2
|
import type { CompletionOrchestrateRequest, CompletionOrchestrateResponse, CompletionStreamRequest, CompletionStreamResponse } from '../schemas/index.js';
|
|
2
3
|
import { type ToolCallbackResult } from '../schemas/completion-stream.js';
|
|
4
|
+
import type { AbortSignalLike } from '../schemas/common.js';
|
|
3
5
|
/**
|
|
4
6
|
* Reads newline-delimited JSON tool results off the duplex request stream
|
|
5
7
|
* and resolves per-callId waiters. Chunk boundaries don't align with lines,
|
|
@@ -10,7 +12,8 @@ import { type ToolCallbackResult } from '../schemas/completion-stream.js';
|
|
|
10
12
|
* and the reader stops holding the request stream open.
|
|
11
13
|
*/
|
|
12
14
|
export declare class ToolResultReader {
|
|
13
|
-
private
|
|
15
|
+
private pending;
|
|
16
|
+
private pendingBytes;
|
|
14
17
|
private readonly results;
|
|
15
18
|
private readonly waiters;
|
|
16
19
|
private failed;
|
|
@@ -18,9 +21,11 @@ export declare class ToolResultReader {
|
|
|
18
21
|
private readonly iterator;
|
|
19
22
|
constructor(inputStream: AsyncIterable<Buffer>);
|
|
20
23
|
private pump;
|
|
24
|
+
private append;
|
|
21
25
|
private failWaiters;
|
|
22
26
|
private accept;
|
|
23
|
-
|
|
27
|
+
/** Rejects with `InferenceCancelledError` when `signal` aborts before the result arrives. */
|
|
28
|
+
waitFor(callId: string, timeoutMs?: number, signal?: AbortSignalLike): Promise<ToolCallbackResult>;
|
|
24
29
|
/**
|
|
25
30
|
* Stop the pump and release the upstream iterator. Idempotent. Pending
|
|
26
31
|
* waiters are failed so a caller blocked in `waitFor` can't hang after the
|
|
@@ -37,5 +42,12 @@ type RunTurn = (request: CompletionStreamRequest) => AsyncGenerator<CompletionSt
|
|
|
37
42
|
* with the extended history.
|
|
38
43
|
*/
|
|
39
44
|
export declare function orchestrateCompletion(request: CompletionOrchestrateRequest, runTurn: RunTurn, reader: Pick<ToolResultReader, 'waitFor'>): AsyncGenerator<CompletionOrchestrateResponse>;
|
|
45
|
+
/**
|
|
46
|
+
* Waits for a tool result as a request of its own, under the completion's
|
|
47
|
+
* `(sessionId, requestId)`, so the registry sees the completion between its
|
|
48
|
+
* turns: a cancel by id or by model, the transport signal and a session close
|
|
49
|
+
* stop the wait, and a session close waits for the completion to end.
|
|
50
|
+
*/
|
|
51
|
+
export declare function registeredToolWaits(reader: Pick<ToolResultReader, 'waitFor'>, request: Pick<CompletionOrchestrateRequest, 'requestId' | 'sessionId' | 'modelId'>): Pick<ToolResultReader, 'waitFor'>;
|
|
40
52
|
export declare function handleCompletionOrchestrate(request: CompletionOrchestrateRequest, inputStream: AsyncIterable<Buffer>): AsyncGenerator<CompletionOrchestrateResponse>;
|
|
41
53
|
export {};
|