@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -50,6 +50,9 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import fs from 'bare-fs';
|
|
54
|
+
import Buffer from 'bare-buffer';
|
|
55
|
+
import { requestParent } from '../../schemas/plugin.js';
|
|
53
56
|
import { getModel } from '../../runtime/model-registry.js';
|
|
54
57
|
import { embedParamsSchema } from '../../schemas/index.js';
|
|
55
58
|
import { buildUnaryResult } from '../../profiling/model-execution.js';
|
|
@@ -58,25 +61,35 @@ import { nowMs } from '../../profiling/index.js';
|
|
|
58
61
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
59
62
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
60
63
|
import { getEngineLogger } from '../../logging/index.js';
|
|
61
|
-
|
|
64
|
+
const modelsWithProjector = new WeakSet();
|
|
65
|
+
export function markEmbeddingProjector(model) {
|
|
66
|
+
modelsWithProjector.add(model);
|
|
67
|
+
}
|
|
68
|
+
export async function embed(params, requestId, sessionId, context) {
|
|
62
69
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
63
70
|
try {
|
|
64
|
-
const
|
|
71
|
+
const parsed = embedParamsSchema.parse(params);
|
|
72
|
+
const inputs = parsed.input !== undefined ? [parsed.input] : parsed.inputs;
|
|
73
|
+
const mediaInputs = inputs?.map((input) => {
|
|
74
|
+
if (typeof input === 'object' && !Array.isArray(input)) {
|
|
75
|
+
throw new EmbedFailedError('This model accepts text and ordered media parts, not raw RGB image inputs');
|
|
76
|
+
}
|
|
77
|
+
return input;
|
|
78
|
+
});
|
|
79
|
+
const { modelId } = parsed;
|
|
65
80
|
// Open a request-scoped lifecycle. The registry routes
|
|
66
81
|
// `cancel({ requestId })` and broad `cancel({ modelId, kind: "embeddings" })`
|
|
67
82
|
// straight to this context's signal. Falls back to a generated
|
|
68
83
|
// id if the caller didn't send one.
|
|
69
84
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
70
85
|
requestId: requestId ?? generateRandomRequestId(),
|
|
86
|
+
sessionId,
|
|
87
|
+
...requestParent(context),
|
|
71
88
|
kind: 'embeddings',
|
|
72
89
|
modelId
|
|
73
|
-
})
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
// shape, even when this op has no per-step emits beyond the
|
|
77
|
-
// registry's own lifecycle lines. Future addon-level warns inside
|
|
78
|
-
// this body should route through `requestLogger`.
|
|
79
|
-
, true);
|
|
90
|
+
}), true);
|
|
91
|
+
if (ctx.signal.aborted)
|
|
92
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
80
93
|
// `requestLogger` is intentionally referenced once so the
|
|
81
94
|
// request-scoped logger is built at handler entry per the canonical
|
|
82
95
|
// shape, even when this op has no per-step emits beyond the
|
|
@@ -84,42 +97,47 @@ export async function embed(params, requestId) {
|
|
|
84
97
|
// this body should route through `requestLogger`.
|
|
85
98
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
86
99
|
const model = getModel(modelId);
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
100
|
+
const hasMedia = mediaInputs?.some((input) => Array.isArray(input) && input.some((part) => typeof part !== 'string'));
|
|
101
|
+
if (hasMedia && !modelsWithProjector.has(model)) {
|
|
102
|
+
throw new EmbedFailedError('Media inputs require modelConfig.projectionModelSrc when loading the embedding model');
|
|
103
|
+
}
|
|
104
|
+
const input = mediaInputs !== undefined
|
|
105
|
+
? await prepareEmbedInputs(mediaInputs, ctx.signal, ctx.requestId)
|
|
106
|
+
: parsed.text;
|
|
107
|
+
if (ctx.signal.aborted)
|
|
108
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
109
|
+
// Admission keeps model-wide cancellation scoped to this request. Media
|
|
110
|
+
// preparation has no native job to cancel, so wire the addon only afterward.
|
|
111
|
+
let cancelPromise;
|
|
96
112
|
const onAbort = () => {
|
|
97
113
|
const addon = model.addon;
|
|
98
114
|
if (addon?.cancel) {
|
|
99
|
-
addon.cancel.call(addon).catch((err) => {
|
|
115
|
+
cancelPromise ??= addon.cancel.call(addon).catch((err) => {
|
|
100
116
|
requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
101
117
|
});
|
|
102
118
|
}
|
|
103
119
|
};
|
|
104
120
|
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
105
|
-
|
|
106
|
-
// register time. The registry may abort synchronously when a
|
|
107
|
-
// cancel-before-begin race resolves or the parent signal was pre-
|
|
108
|
-
// aborted — re-use `onAbort` so the listener body is the single
|
|
109
|
-
// source of truth for "what cancel does."
|
|
110
|
-
if (ctx.signal.aborted)
|
|
111
|
-
onAbort();
|
|
112
|
-
ctx.scope.defer(() => {
|
|
121
|
+
ctx.scope.defer(async () => {
|
|
113
122
|
ctx.signal.removeEventListener('abort', onAbort);
|
|
123
|
+
// A delayed native cancel must finish before the next job owns this model slot.
|
|
124
|
+
await cancelPromise;
|
|
114
125
|
});
|
|
115
126
|
const modelStart = nowMs();
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
127
|
+
let response;
|
|
128
|
+
let rawEmbeddings;
|
|
129
|
+
try {
|
|
130
|
+
response = (await model.run(input));
|
|
131
|
+
if (ctx.signal.aborted)
|
|
132
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
133
|
+
rawEmbeddings = await response.await();
|
|
134
|
+
if (ctx.signal.aborted)
|
|
135
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
119
136
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
137
|
+
catch (cause) {
|
|
138
|
+
if (ctx.signal.aborted)
|
|
139
|
+
throw new InferenceCancelledError(ctx.requestId, {}, cause);
|
|
140
|
+
throw cause;
|
|
123
141
|
}
|
|
124
142
|
const modelExecutionMs = nowMs() - modelStart;
|
|
125
143
|
const stats = {
|
|
@@ -134,7 +152,7 @@ export async function embed(params, requestId) {
|
|
|
134
152
|
...(response.stats?.context_size !== undefined && { contextSize: response.stats.context_size })
|
|
135
153
|
};
|
|
136
154
|
const embeddingsArray = rawEmbeddings[0];
|
|
137
|
-
if (Array.isArray(text)) {
|
|
155
|
+
if (parsed.input === undefined && (parsed.inputs !== undefined || Array.isArray(parsed.text))) {
|
|
138
156
|
if (!embeddingsArray || embeddingsArray.length === 0) {
|
|
139
157
|
throw new EmbedNoEmbeddingsError();
|
|
140
158
|
}
|
|
@@ -191,3 +209,40 @@ export function normalizeVector(vector) {
|
|
|
191
209
|
}
|
|
192
210
|
return normalizedVector;
|
|
193
211
|
}
|
|
212
|
+
async function prepareEmbedInputs(inputs, signal, requestId) {
|
|
213
|
+
const result = [];
|
|
214
|
+
for (const input of inputs) {
|
|
215
|
+
if (signal.aborted)
|
|
216
|
+
throw new InferenceCancelledError(requestId);
|
|
217
|
+
if (typeof input === 'string') {
|
|
218
|
+
result.push(input);
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
const parts = [];
|
|
222
|
+
for (const part of input) {
|
|
223
|
+
if (signal.aborted)
|
|
224
|
+
throw new InferenceCancelledError(requestId);
|
|
225
|
+
if (typeof part === 'string') {
|
|
226
|
+
parts.push(part);
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
const { type, value } = part.media;
|
|
230
|
+
try {
|
|
231
|
+
const media = type === 'base64'
|
|
232
|
+
? Buffer.from(value, 'base64')
|
|
233
|
+
: await fs.promises.readFile(value, 'buffer');
|
|
234
|
+
parts.push({ media });
|
|
235
|
+
}
|
|
236
|
+
catch (cause) {
|
|
237
|
+
if (signal.aborted)
|
|
238
|
+
throw new InferenceCancelledError(requestId, {}, cause);
|
|
239
|
+
throw new EmbedFailedError('Unable to read embedding media', cause);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
result.push(parts);
|
|
244
|
+
}
|
|
245
|
+
if (signal.aborted)
|
|
246
|
+
throw new InferenceCancelledError(requestId);
|
|
247
|
+
return { inputs: result };
|
|
248
|
+
}
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { type CacheMessage } from '../../utils/index.js';
|
|
2
|
+
import type { Tool } from '../../schemas/tools.js';
|
|
2
3
|
export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
|
|
3
4
|
export declare function generateConfigHash(systemPrompt: string | null): string;
|
|
4
|
-
export declare function generateCacheKey(messages: CacheMessage[]): string;
|
|
5
|
+
export declare function generateCacheKey(messages: CacheMessage[], tools?: readonly Tool[]): string;
|
|
6
|
+
export declare function resolveCacheFilePath(modelId: string, configHash: string, cacheKey: string): string;
|
|
5
7
|
export declare function getCacheFilePath(modelId: string, configHash: string, cacheKey: string): Promise<string>;
|
|
6
|
-
export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
|
|
8
|
+
export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[], tools?: readonly Tool[]): Promise<{
|
|
7
9
|
cacheKey: string;
|
|
8
10
|
cachePath: string;
|
|
9
11
|
} | null>;
|
|
10
|
-
export declare function getCurrentCacheInfo(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
|
|
12
|
+
export declare function getCurrentCacheInfo(modelId: string, configHash: string, currentHistory: CacheMessage[], tools?: readonly Tool[]): Promise<{
|
|
11
13
|
cacheKey: string;
|
|
12
14
|
cachePath: string;
|
|
13
15
|
}>;
|
|
@@ -5,14 +5,16 @@ import { getAutoCacheLookupHistory, getKVCacheDir, validateAndJoinPath } from '.
|
|
|
5
5
|
import { getEngineLogger } from '../../logging/index.js';
|
|
6
6
|
import { PathTraversalError } from '../../errors/index.js';
|
|
7
7
|
import { Buffer } from 'bare-buffer';
|
|
8
|
-
import { markAutoCacheKey } from '
|
|
8
|
+
import { markAutoCacheKey } from '../../plugins/ops/kv-cache-retention.js';
|
|
9
9
|
const logger = getEngineLogger();
|
|
10
10
|
// In-memory KV-cache state lives in `KvCacheSession`, its single mutation
|
|
11
11
|
// point. This module keeps only the pure path / hash utilities that don't
|
|
12
12
|
// touch in-memory state.
|
|
13
13
|
export function extractSystemPrompt(messages) {
|
|
14
14
|
const systemMessage = messages.find((msg) => msg.role === 'system');
|
|
15
|
-
|
|
15
|
+
if (!systemMessage || typeof systemMessage.content !== 'string')
|
|
16
|
+
return null;
|
|
17
|
+
return systemMessage.content;
|
|
16
18
|
}
|
|
17
19
|
// Names the cache file by system prompt only. Tools stay out on purpose: the
|
|
18
20
|
// addon compares the rendered prompt against the tokens the file holds, so a
|
|
@@ -24,15 +26,22 @@ export function generateConfigHash(systemPrompt) {
|
|
|
24
26
|
hash.update(Buffer.from(JSON.stringify({ systemPrompt, tools: [] }), 'utf8'));
|
|
25
27
|
return hash.digest('hex').substring(0, 16);
|
|
26
28
|
}
|
|
27
|
-
export function generateCacheKey(messages) {
|
|
29
|
+
export function generateCacheKey(messages, tools) {
|
|
28
30
|
const hash = crypto.createHash('sha-256');
|
|
29
|
-
|
|
31
|
+
// Before the first assistant reply, the rendered tools are part of the
|
|
32
|
+
// reusable prefix. Keep those primes isolated; established conversations
|
|
33
|
+
// retain history-only keys so the addon can trim a changed tool block.
|
|
34
|
+
const historyString = JSON.stringify(tools?.length && !messages.some((message) => message.role === 'assistant')
|
|
35
|
+
? { messages, tools }
|
|
36
|
+
: messages);
|
|
30
37
|
const historyBuffer = Buffer.from(historyString, 'utf8');
|
|
31
38
|
hash.update(historyBuffer);
|
|
32
39
|
const hashString = hash.digest('hex');
|
|
33
40
|
return hashString.substring(0, 16);
|
|
34
41
|
}
|
|
35
|
-
|
|
42
|
+
// Resolves the path without creating its directories, for a cache that may
|
|
43
|
+
// not exist (a seed source).
|
|
44
|
+
export function resolveCacheFilePath(modelId, configHash, cacheKey) {
|
|
36
45
|
const cacheDir = getKVCacheDir();
|
|
37
46
|
const sessionCacheDir = validateAndJoinPath(cacheDir, cacheKey);
|
|
38
47
|
const modelCacheDir = validateAndJoinPath(sessionCacheDir, modelId);
|
|
@@ -50,12 +59,14 @@ export async function getCacheFilePath(modelId, configHash, cacheKey) {
|
|
|
50
59
|
return cachePath;
|
|
51
60
|
}
|
|
52
61
|
// Used for auto-generated cache key
|
|
53
|
-
export async function findMatchingCache(modelId, configHash, currentHistory) {
|
|
54
|
-
if (currentHistory.length
|
|
62
|
+
export async function findMatchingCache(modelId, configHash, currentHistory, tools) {
|
|
63
|
+
if (currentHistory.length === 0) {
|
|
55
64
|
return null;
|
|
56
65
|
}
|
|
57
66
|
const previousHistory = getAutoCacheLookupHistory(currentHistory);
|
|
58
|
-
|
|
67
|
+
if (previousHistory.length === 0 && !tools?.length)
|
|
68
|
+
return null;
|
|
69
|
+
const cacheKey = generateCacheKey(previousHistory, tools);
|
|
59
70
|
const cachePath = resolveCacheFilePath(modelId, configHash, cacheKey);
|
|
60
71
|
try {
|
|
61
72
|
await fsPromises.access(cachePath);
|
|
@@ -66,8 +77,8 @@ export async function findMatchingCache(modelId, configHash, currentHistory) {
|
|
|
66
77
|
return null;
|
|
67
78
|
}
|
|
68
79
|
}
|
|
69
|
-
export async function getCurrentCacheInfo(modelId, configHash, currentHistory) {
|
|
70
|
-
const cacheKey = generateCacheKey(currentHistory);
|
|
80
|
+
export async function getCurrentCacheInfo(modelId, configHash, currentHistory, tools) {
|
|
81
|
+
const cacheKey = generateCacheKey(currentHistory, tools);
|
|
71
82
|
await markAutoCacheKey(cacheKey);
|
|
72
83
|
const cachePath = await getCacheFilePath(modelId, configHash, cacheKey);
|
|
73
84
|
return { cacheKey, cachePath };
|
|
@@ -1,8 +1,16 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
1
2
|
import { type LoadModelServerParams } from '../../schemas/index.js';
|
|
2
3
|
import type { LoadTimingStats } from '../../profiling/types.js';
|
|
3
4
|
export interface LoadModelResult {
|
|
4
5
|
timing?: LoadTimingStats;
|
|
5
6
|
}
|
|
6
|
-
export
|
|
7
|
+
export interface LoadModelOptions {
|
|
7
8
|
collectTiming?: boolean;
|
|
8
|
-
|
|
9
|
+
request?: {
|
|
10
|
+
signal: AbortSignal;
|
|
11
|
+
requestId: string;
|
|
12
|
+
};
|
|
13
|
+
/** Session that takes a lease on the model. Defaults to the owner session. */
|
|
14
|
+
sessionId?: string;
|
|
15
|
+
}
|
|
16
|
+
export declare function loadModel(params: LoadModelServerParams, options?: LoadModelOptions): Promise<LoadModelResult>;
|
|
@@ -1,26 +1,46 @@
|
|
|
1
1
|
import { loadModelServerParamsSchema, normalizeModelType } from '../../schemas/index.js';
|
|
2
2
|
import { nowMs } from '../../profiling/clock.js';
|
|
3
3
|
import { forwardBackendDiagnostics } from '../../profiling/backend-diagnostics.js';
|
|
4
|
-
import { isModelLoaded, registerModel } from '../../runtime/model-registry.js';
|
|
4
|
+
import { addModelLease, getModelEntry, isModelLoaded, isModelReconfigured, registerModel } from '../../runtime/model-registry.js';
|
|
5
|
+
import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
|
|
6
|
+
import { withModelWorkLock } from '../../runtime/model-work-lock.js';
|
|
5
7
|
import { startLogBuffering, stopLogBufferingWithTimeout } from '../../runtime/logging-stream-registry.js';
|
|
6
|
-
import { detectShardedModel,
|
|
7
|
-
import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError } from '../../errors/index.js';
|
|
8
|
-
import { ensureAddonLoggerReady, getPlugin } from '
|
|
9
|
-
import {
|
|
8
|
+
import { detectShardedModel, checkAllShardsExist } from '../../utils/index.js';
|
|
9
|
+
import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError, ModelReconfiguredError, InferenceCancelledError } from '../../errors/index.js';
|
|
10
|
+
import { ensureAddonLoggerReady, getPlugin } from '../../plugins/index.js';
|
|
11
|
+
import { runLoadFitCheck } from '../../model-fit/load-check.js';
|
|
10
12
|
import { refusesLoad, resolveModelFitPolicy, runsProbe } from '../../schemas/model-fit-policy.js';
|
|
11
13
|
import { getConfig } from '../../runtime/state.js';
|
|
14
|
+
import { resolveToolDialect } from '../../utils/tools/dialect.js';
|
|
12
15
|
import { promises as fsPromises } from 'bare-fs';
|
|
13
16
|
import path from 'bare-path';
|
|
14
17
|
import { getEngineLogger } from '../../logging/index.js';
|
|
15
18
|
const logger = getEngineLogger();
|
|
16
19
|
export async function loadModel(params, options) {
|
|
17
|
-
const
|
|
20
|
+
const parsed = loadModelServerParamsSchema.parse(params);
|
|
21
|
+
return withModelWorkLock(parsed.modelId, () => loadModelLocked(parsed, options), options?.request);
|
|
22
|
+
}
|
|
23
|
+
async function loadModelLocked(params, options) {
|
|
24
|
+
const { modelId, modelPath, options: modelOptions, artifacts, modelName } = params;
|
|
18
25
|
const { modelConfig, modelType: rawModelType, modelFitPolicy } = modelOptions;
|
|
19
26
|
// Normalize modelType to canonical form (handles aliases and custom types)
|
|
20
27
|
const modelType = normalizeModelType(rawModelType);
|
|
21
|
-
|
|
28
|
+
const sessionId = options?.sessionId ?? OWNER_SESSION_ID;
|
|
29
|
+
// A cancel that landed while this load waited for another load of the same
|
|
30
|
+
// model must not take a lease the caller no longer wants.
|
|
31
|
+
if (options?.request?.signal.aborted === true) {
|
|
32
|
+
throw new InferenceCancelledError(options.request.requestId);
|
|
33
|
+
}
|
|
34
|
+
// An already loaded model is shared: the caller takes a lease on it. A
|
|
35
|
+
// session that holds no lease on a reconfigured model is refused: it asked
|
|
36
|
+
// for the configuration the id hashes, which the model no longer has.
|
|
22
37
|
if (isModelLoaded(modelId)) {
|
|
23
|
-
|
|
38
|
+
const entry = getModelEntry(modelId);
|
|
39
|
+
if (entry && !entry.leases.has(sessionId) && isModelReconfigured(modelId)) {
|
|
40
|
+
throw new ModelReconfiguredError(modelId);
|
|
41
|
+
}
|
|
42
|
+
addModelLease(modelId, sessionId);
|
|
43
|
+
logger.info(`${modelType} model ${modelId} is already loaded; lease added for ${sessionId}`);
|
|
24
44
|
return {};
|
|
25
45
|
}
|
|
26
46
|
// Detect if sharded model
|
|
@@ -32,12 +52,9 @@ export async function loadModel(params, options) {
|
|
|
32
52
|
throw new PluginNotFoundError(modelType);
|
|
33
53
|
}
|
|
34
54
|
if (isShardedModel) {
|
|
35
|
-
// For sharded models, validate all shards and tensors.txt exist
|
|
36
55
|
const shardDir = path.dirname(modelPath);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const numberedShards = generateShardFilenames(modelFileName);
|
|
40
|
-
throw new ModelFileNotFoundError(`Missing shards or ${shardInfo.baseFilename}.tensors.txt. Expected ${numberedShards.length} shard files + tensors.txt in ${shardDir}`);
|
|
56
|
+
if (!(await checkAllShardsExist(shardDir, modelFileName))) {
|
|
57
|
+
throw new ModelFileNotFoundError(`Missing shards. Expected ${shardInfo.totalShards} shard files in ${shardDir}`);
|
|
41
58
|
}
|
|
42
59
|
}
|
|
43
60
|
else if (!plugin.skipPrimaryModelPathValidation) {
|
|
@@ -59,18 +76,23 @@ export async function loadModel(params, options) {
|
|
|
59
76
|
// the real load uses, and before `createModel()` so it never competes with
|
|
60
77
|
// the native load for device memory.
|
|
61
78
|
const fitPolicy = resolveModelFitPolicy(modelFitPolicy, getConfig().modelFitPolicy);
|
|
62
|
-
const
|
|
63
|
-
|
|
79
|
+
const request = options?.request;
|
|
80
|
+
const fitCheck = runsProbe(fitPolicy)
|
|
81
|
+
? await runLoadFitCheck({
|
|
64
82
|
modelId,
|
|
65
83
|
modelType: modelType,
|
|
66
84
|
modelPath,
|
|
67
85
|
modelConfig,
|
|
68
86
|
artifacts,
|
|
69
87
|
isShardedModel
|
|
70
|
-
})
|
|
88
|
+
}, request !== undefined ? { signal: request.signal } : {})
|
|
71
89
|
: undefined;
|
|
90
|
+
const fitProbe = fitCheck?.probe;
|
|
91
|
+
if (request?.signal.aborted === true) {
|
|
92
|
+
throw new InferenceCancelledError(request.requestId);
|
|
93
|
+
}
|
|
72
94
|
if (refusesLoad(fitPolicy, fitProbe?.verdict)) {
|
|
73
|
-
throw new ModelFitRefusedError(modelType,
|
|
95
|
+
throw new ModelFitRefusedError(modelType, 'does-not-fit', fitCheck?.detail);
|
|
74
96
|
}
|
|
75
97
|
// Load the addon's logger before the addon itself runs. Plugins that defer
|
|
76
98
|
// their logging module do so to keep registration free of native loads, so
|
|
@@ -90,14 +112,18 @@ export async function loadModel(params, options) {
|
|
|
90
112
|
await result.model.load(false);
|
|
91
113
|
const modelInitializationTimeMs = options?.collectTiming ? nowMs() - initStart : undefined;
|
|
92
114
|
logger.info(`${modelType} model ${modelId} loaded`);
|
|
115
|
+
const toolDialect = 'completionStream' in plugin.handlers
|
|
116
|
+
? await resolveToolDialect(modelName, modelPath)
|
|
117
|
+
: undefined;
|
|
93
118
|
registerModel(modelId, {
|
|
94
119
|
model: result.model,
|
|
95
120
|
path: modelPath,
|
|
96
121
|
config: modelConfig,
|
|
97
122
|
modelType: modelType,
|
|
98
123
|
name: modelName,
|
|
99
|
-
fitProbe
|
|
100
|
-
|
|
124
|
+
fitProbe,
|
|
125
|
+
toolDialect
|
|
126
|
+
}, sessionId);
|
|
101
127
|
const loadResult = modelInitializationTimeMs !== undefined ? { timing: { modelInitializationTimeMs } } : {};
|
|
102
128
|
return forwardBackendDiagnostics(loadResult, result);
|
|
103
129
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../schemas/plugin.js';
|
|
1
2
|
import { type TranscribeParams, type TranscribeSegment, type TranscribeStats, type TranscribeStreamEvent, type ParakeetStreamingRunConfig } from '../../schemas/index.js';
|
|
2
3
|
import type Buffer from 'bare-buffer';
|
|
3
4
|
import type { InferenceBackendDiagnostics } from '../../schemas/index.js';
|
|
@@ -10,13 +11,13 @@ type TranscribeReturn = {
|
|
|
10
11
|
};
|
|
11
12
|
export declare function transcribe(params: TranscribeParams & {
|
|
12
13
|
metadata: true;
|
|
13
|
-
}, requestId?: string): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
|
|
14
|
-
export declare function transcribe(params: TranscribeParams, requestId?: string): AsyncGenerator<string, TranscribeReturn, void>;
|
|
14
|
+
}, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
|
|
15
|
+
export declare function transcribe(params: TranscribeParams, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string, TranscribeReturn, void>;
|
|
15
16
|
export interface TranscribeStreamOpts {
|
|
16
17
|
emitVadEvents?: boolean;
|
|
17
18
|
endOfTurnSilenceMs?: number;
|
|
18
19
|
vadRunIntervalMs?: number;
|
|
19
20
|
parakeetStreamingConfig?: ParakeetStreamingRunConfig;
|
|
20
21
|
}
|
|
21
|
-
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
22
|
-
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
22
|
+
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
23
|
+
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
@@ -50,11 +50,13 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import { requestParent } from '../../schemas/plugin.js';
|
|
53
54
|
import { getModel, getModelConfig, getModelEntry } from '../../runtime/model-registry.js';
|
|
54
55
|
import { ModelType } from '../../schemas/index.js';
|
|
55
56
|
import { createAudioStream } from '../../utils/audio-input.js';
|
|
57
|
+
import { untilAborted } from '../../utils/until-aborted.js';
|
|
56
58
|
import { getEngineLogger } from '../../logging/index.js';
|
|
57
|
-
import { TranscriptionFailedError } from '../../errors/index.js';
|
|
59
|
+
import { InferenceCancelledError, TranscriptionFailedError } from '../../errors/index.js';
|
|
58
60
|
import { nowMs } from '../../profiling/index.js';
|
|
59
61
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
60
62
|
import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
|
|
@@ -62,8 +64,11 @@ import { assertMetadataSupported, toTranscribeSegment } from '../../utils/transc
|
|
|
62
64
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
63
65
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
64
66
|
import { isEndOfTurnEvent, isVadEvent, toEndOfTurnEvent, toVadStateEvent } from '../../utils/asr-events.js';
|
|
65
|
-
import { buildWhisperReloadConfig } from '
|
|
67
|
+
import { buildWhisperReloadConfig } from '../../plugins/builtin/asr-ggml/config.js';
|
|
66
68
|
export { assertMetadataSupported, toTranscribeSegment };
|
|
69
|
+
async function* iterateAll(source) {
|
|
70
|
+
yield* source;
|
|
71
|
+
}
|
|
67
72
|
function hasRunStreaming(model) {
|
|
68
73
|
return 'runStreaming' in model && typeof model.runStreaming === 'function';
|
|
69
74
|
}
|
|
@@ -173,10 +178,23 @@ function mapTranscribeStats(stats) {
|
|
|
173
178
|
}),
|
|
174
179
|
...(stats?.encoderUsedCoreml !== undefined && {
|
|
175
180
|
encoderUsedCoreml: stats.encoderUsedCoreml
|
|
181
|
+
}),
|
|
182
|
+
...(stats?.aoscActive !== undefined && { aoscActive: stats.aoscActive }),
|
|
183
|
+
// MOSS-Transcribe-Diarize reports its counters under its own names. Its
|
|
184
|
+
// stage timings land on the shared encoder/decoder fields and its segment
|
|
185
|
+
// count on `totalSegments`; the token counts and prefill are MOSS-only.
|
|
186
|
+
...(stats?.segments !== undefined && { totalSegments: stats.segments }),
|
|
187
|
+
...(stats?.encodeMs !== undefined && { encoderTime: stats.encodeMs }),
|
|
188
|
+
...(stats?.decodeMs !== undefined && { decoderTime: stats.decodeMs }),
|
|
189
|
+
...(stats?.prefillMs !== undefined && { prefillTime: stats.prefillMs }),
|
|
190
|
+
...(stats?.audioTokens !== undefined && { audioTokens: stats.audioTokens }),
|
|
191
|
+
...(stats?.promptTokens !== undefined && { promptTokens: stats.promptTokens }),
|
|
192
|
+
...(stats?.generatedTokens !== undefined && {
|
|
193
|
+
generatedTokens: stats.generatedTokens
|
|
176
194
|
})
|
|
177
195
|
};
|
|
178
196
|
}
|
|
179
|
-
export async function* transcribe(params, requestId) {
|
|
197
|
+
export async function* transcribe(params, requestId, sessionId, context) {
|
|
180
198
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
181
199
|
try {
|
|
182
200
|
const { modelId, metadata } = params;
|
|
@@ -186,9 +204,18 @@ export async function* transcribe(params, requestId) {
|
|
|
186
204
|
// caller didn't send one.
|
|
187
205
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
188
206
|
requestId: requestId ?? generateRandomRequestId(),
|
|
207
|
+
sessionId,
|
|
208
|
+
...requestParent(context),
|
|
189
209
|
kind: 'transcribe',
|
|
190
210
|
modelId
|
|
191
|
-
})
|
|
211
|
+
})
|
|
212
|
+
// A request cancelled while it queued holds no slot; the model-wide
|
|
213
|
+
// `addon.cancel()` below would stop the job that does.
|
|
214
|
+
, true);
|
|
215
|
+
// A request cancelled while it queued holds no slot; the model-wide
|
|
216
|
+
// `addon.cancel()` below would stop the job that does.
|
|
217
|
+
if (ctx.signal.aborted)
|
|
218
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
192
219
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
193
220
|
const engineType = getEngineModelType(modelId);
|
|
194
221
|
const hasMossOptions = params.hotwords !== undefined || params.maxNewTokens !== undefined;
|
|
@@ -248,9 +275,7 @@ export async function* transcribe(params, requestId) {
|
|
|
248
275
|
const chunks = (Array.isArray(output) ? output : [output]);
|
|
249
276
|
if (metadata) {
|
|
250
277
|
for (const chunk of chunks) {
|
|
251
|
-
if (!chunk
|
|
252
|
-
continue;
|
|
253
|
-
if (silenceMarker && chunk.text.includes(silenceMarker))
|
|
278
|
+
if (!carriesSegment(chunk, silenceMarker))
|
|
254
279
|
continue;
|
|
255
280
|
yield toTranscribeSegment(chunk);
|
|
256
281
|
}
|
|
@@ -297,7 +322,7 @@ function buildRunStreamingOpts(engineType, opts) {
|
|
|
297
322
|
}
|
|
298
323
|
return runOpts;
|
|
299
324
|
}
|
|
300
|
-
export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId) {
|
|
325
|
+
export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId, sessionId, context) {
|
|
301
326
|
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
302
327
|
try {
|
|
303
328
|
// Same `kind: "transcribe"` as the unary variant — the registry
|
|
@@ -306,9 +331,13 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
306
331
|
// either shape.
|
|
307
332
|
const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
|
|
308
333
|
requestId: requestId ?? generateRandomRequestId(),
|
|
334
|
+
sessionId,
|
|
335
|
+
...requestParent(context),
|
|
309
336
|
kind: 'transcribe',
|
|
310
337
|
modelId
|
|
311
338
|
}), true);
|
|
339
|
+
if (ctx.signal.aborted)
|
|
340
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
312
341
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
313
342
|
const engineType = getEngineModelType(modelId);
|
|
314
343
|
assertMetadataSupported(modelId, engineType, metadata);
|
|
@@ -321,26 +350,19 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
321
350
|
if (!hasRunStreaming(model)) {
|
|
322
351
|
throw new TranscriptionFailedError(`Model ${modelId} does not support streaming transcription`);
|
|
323
352
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
};
|
|
332
|
-
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
333
|
-
if (ctx.signal.aborted)
|
|
334
|
-
onAbort();
|
|
335
|
-
ctx.scope.defer(() => {
|
|
336
|
-
ctx.signal.removeEventListener('abort', onAbort);
|
|
337
|
-
});
|
|
353
|
+
// A streaming session ends when its input ends, so an abort ends the input
|
|
354
|
+
// and the session drains the audio it already has, yielding nothing more.
|
|
355
|
+
// The model-wide `addon.cancel()` the unary path uses is not safe here: a
|
|
356
|
+
// parakeet stream cancelled that way leaves its stream state to the next
|
|
357
|
+
// session, whose output starts with the cancelled session's audio and never
|
|
358
|
+
// ends.
|
|
359
|
+
const input = untilAborted(iterateAll(audioInputStream), ctx.signal);
|
|
338
360
|
const runOpts = buildRunStreamingOpts(engineType, opts);
|
|
339
361
|
const modelStart = nowMs();
|
|
340
|
-
const response = await model.runStreaming(
|
|
362
|
+
const response = await model.runStreaming(input, runOpts);
|
|
341
363
|
for await (const output of response.iterate()) {
|
|
342
364
|
if (ctx.signal.aborted)
|
|
343
|
-
|
|
365
|
+
continue;
|
|
344
366
|
requestLogger.debug('Live Transcription Update:', output);
|
|
345
367
|
if (!Array.isArray(output)) {
|
|
346
368
|
if (isVadEvent(output)) {
|
|
@@ -386,17 +408,30 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
386
408
|
await result_2;
|
|
387
409
|
}
|
|
388
410
|
}
|
|
411
|
+
/**
|
|
412
|
+
* Whether a native segment becomes a metadata segment. Text-less segments are
|
|
413
|
+
* dropped, except an explicit final marker: parakeet streaming closes a chunk
|
|
414
|
+
* with `isFinal: true` and no text, and that marker is the only finality signal
|
|
415
|
+
* the caller gets for the chunk.
|
|
416
|
+
*/
|
|
417
|
+
function carriesSegment(segment, silenceMarker) {
|
|
418
|
+
if (!segment.text)
|
|
419
|
+
return segment.isFinal === true;
|
|
420
|
+
return !silenceMarker || !segment.text.includes(silenceMarker);
|
|
421
|
+
}
|
|
389
422
|
function* emitSegment(segment, metadata, silenceMarker) {
|
|
423
|
+
if (metadata) {
|
|
424
|
+
if (carriesSegment(segment, silenceMarker))
|
|
425
|
+
yield toTranscribeSegment(segment);
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
390
428
|
if (!segment.text) {
|
|
391
429
|
return;
|
|
392
430
|
}
|
|
393
431
|
if (silenceMarker && segment.text.includes(silenceMarker)) {
|
|
394
432
|
return;
|
|
395
433
|
}
|
|
396
|
-
if (
|
|
397
|
-
yield toTranscribeSegment(segment);
|
|
398
|
-
}
|
|
399
|
-
else if (segment.text.trim()) {
|
|
434
|
+
if (segment.text.trim()) {
|
|
400
435
|
yield segment.text;
|
|
401
436
|
}
|
|
402
437
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../schemas/plugin.js';
|
|
1
2
|
import { type TranslateParams, type TranslationStats } from '../../schemas/index.js';
|
|
2
3
|
export declare function getLanguage(code: string | undefined): string;
|
|
3
4
|
export declare function isAfrican(code: string | undefined): boolean;
|
|
4
|
-
export declare function translate(params: TranslateParams, requestId?: string): AsyncGenerator<string, {
|
|
5
|
+
export declare function translate(params: TranslateParams, requestId?: string, sessionId?: string, invoke?: PluginHandlerContext): AsyncGenerator<string, {
|
|
5
6
|
modelExecutionMs: number;
|
|
6
7
|
stats?: TranslationStats;
|
|
7
8
|
}, unknown>;
|