@qvac/sdk 0.10.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +1455 -29
- package/README.md +2 -0
- package/dist/client/api/cancel.d.ts +24 -32
- package/dist/client/api/cancel.js +58 -31
- package/dist/client/api/classify.d.ts +23 -0
- package/dist/client/api/classify.js +43 -0
- package/dist/client/api/client-request-id.d.ts +17 -0
- package/dist/client/api/client-request-id.js +27 -0
- package/dist/client/api/completion-stream.d.ts +1 -1
- package/dist/client/api/completion-stream.js +34 -3
- package/dist/client/api/diffusion.d.ts +46 -0
- package/dist/client/api/diffusion.js +50 -1
- package/dist/client/api/download-asset.d.ts +12 -2
- package/dist/client/api/download-asset.js +21 -3
- package/dist/client/api/embed.d.ts +8 -4
- package/dist/client/api/embed.js +14 -1
- package/dist/client/api/get-model-info.d.ts +1 -1
- package/dist/client/api/index.d.ts +5 -0
- package/dist/client/api/index.js +5 -0
- package/dist/client/api/load-model.d.ts +12 -4
- package/dist/client/api/load-model.js +49 -2
- package/dist/client/api/rag.d.ts +9 -3
- package/dist/client/api/rag.js +23 -3
- package/dist/client/api/transcribe.d.ts +23 -8
- package/dist/client/api/transcribe.js +69 -4
- package/dist/client/api/unload-model.d.ts +5 -2
- package/dist/client/api/unload-model.js +9 -4
- package/dist/client/api/upscale.d.ts +45 -0
- package/dist/client/api/upscale.js +99 -0
- package/dist/client/api/video.d.ts +64 -0
- package/dist/client/api/video.js +147 -0
- package/dist/client/api/vla-helpers.d.ts +45 -0
- package/dist/client/api/vla-helpers.js +130 -0
- package/dist/client/api/vla.d.ts +65 -0
- package/dist/client/api/vla.js +109 -0
- package/dist/client/config-loader/resolve-config.node.d.ts +16 -0
- package/dist/client/config-loader/resolve-config.node.js +77 -23
- package/dist/client/plugins-factory.d.ts +8 -0
- package/dist/client/plugins-factory.js +10 -0
- package/dist/client/rpc/bare-client.js +62 -66
- package/dist/client/rpc/ensure-worker-ready.d.ts +9 -0
- package/dist/client/rpc/ensure-worker-ready.js +17 -0
- package/dist/client/rpc/expo-rpc-client.js +2 -1
- package/dist/client/rpc/node-rpc-client.js +22 -8
- package/dist/client/rpc/rpc-client.d.ts +2 -2
- package/dist/client/rpc/rpc-client.js +10 -4
- package/dist/client/rpc/rpc-error.d.ts +9 -0
- package/dist/client/rpc/rpc-error.js +105 -0
- package/dist/commands/bundle/bare-pack.d.ts +13 -0
- package/dist/commands/bundle/bare-pack.js +54 -0
- package/dist/commands/bundle/constants.d.ts +7 -0
- package/dist/commands/bundle/constants.js +23 -0
- package/dist/commands/bundle/entry-gen.d.ts +2 -0
- package/dist/commands/bundle/entry-gen.js +48 -0
- package/dist/commands/bundle/index.d.ts +20 -0
- package/dist/commands/bundle/index.js +141 -0
- package/dist/commands/bundle/manifest.d.ts +21 -0
- package/dist/commands/bundle/manifest.js +193 -0
- package/dist/commands/bundle/plugins.d.ts +9 -0
- package/dist/commands/bundle/plugins.js +51 -0
- package/dist/commands/index.d.ts +5 -0
- package/dist/commands/index.js +2 -0
- package/dist/commands/verify/abi.d.ts +52 -0
- package/dist/commands/verify/abi.js +125 -0
- package/dist/commands/verify/addon-source.d.ts +43 -0
- package/dist/commands/verify/addon-source.js +75 -0
- package/dist/commands/verify/bundle-source.d.ts +12 -0
- package/dist/commands/verify/bundle-source.js +53 -0
- package/dist/commands/verify/index.d.ts +59 -0
- package/dist/commands/verify/index.js +371 -0
- package/dist/commands/verify/node-modules-source.d.ts +11 -0
- package/dist/commands/verify/node-modules-source.js +76 -0
- package/dist/commands/verify/prebuilds.d.ts +15 -0
- package/dist/commands/verify/prebuilds.js +39 -0
- package/dist/examples/cancel-by-request-id.d.ts +38 -0
- package/dist/examples/cancel-by-request-id.js +103 -0
- package/dist/examples/classification/classify-image.d.ts +2 -0
- package/dist/examples/classification/classify-image.js +23 -0
- package/dist/examples/diffusion-esrgan-upscale.d.ts +2 -0
- package/dist/examples/diffusion-esrgan-upscale.js +78 -0
- package/dist/examples/diffusion-txt2vid.d.ts +2 -0
- package/dist/examples/diffusion-txt2vid.js +81 -0
- package/dist/examples/download-with-cancel.js +10 -10
- package/dist/examples/embed-multi-gpu.d.ts +2 -0
- package/dist/examples/embed-multi-gpu.js +46 -0
- package/dist/examples/llamacpp-multi-gpu.d.ts +2 -0
- package/dist/examples/llamacpp-multi-gpu.js +57 -0
- package/dist/examples/plugins.js +3 -7
- package/dist/examples/rag/rag-hyperdb/cancellation.js +20 -17
- package/dist/examples/tools/llamacpp-tools-gemma4.d.ts +2 -0
- package/dist/examples/tools/llamacpp-tools-gemma4.js +73 -0
- package/dist/examples/tools/llamacpp-tools-qwen35.d.ts +2 -0
- package/dist/examples/tools/llamacpp-tools-qwen35.js +69 -0
- package/dist/examples/transcription/parakeet-ctc-filesystem.js +18 -12
- package/dist/examples/transcription/parakeet-microphone-record.js +7 -12
- package/dist/examples/transcription/parakeet-microphone-stream.d.ts +2 -0
- package/dist/examples/transcription/parakeet-microphone-stream.js +79 -0
- package/dist/examples/transcription/parakeet-sortformer-streaming.d.ts +2 -0
- package/dist/examples/transcription/parakeet-sortformer-streaming.js +121 -0
- package/dist/examples/transcription/parakeet-sortformer.js +58 -55
- package/dist/examples/transcription/parakeet-tdt-filesystem.js +18 -15
- package/dist/examples/transcription/whispercpp-microphone-conversation.d.ts +2 -0
- package/dist/examples/transcription/whispercpp-microphone-conversation.js +107 -0
- package/dist/examples/tts/chatterbox.js +7 -16
- package/dist/examples/tts/llm-to-tts-streaming.js +3 -10
- package/dist/examples/tts/supertonic-multilingual.js +4 -12
- package/dist/examples/tts/supertonic.js +5 -12
- package/dist/examples/vla-smolvla.d.ts +2 -0
- package/dist/examples/vla-smolvla.js +75 -0
- package/dist/examples/voice-assistant/voice-assistant.js +3 -10
- package/dist/expo/plugins/find-in-ancestor-node-modules.d.ts +14 -0
- package/dist/expo/plugins/find-in-ancestor-node-modules.js +24 -0
- package/dist/expo/plugins/get-project-root.d.ts +22 -0
- package/dist/expo/plugins/get-project-root.js +23 -0
- package/dist/expo/plugins/index.d.ts +4 -0
- package/dist/expo/plugins/index.js +4 -0
- package/dist/expo/plugins/resolve-sdk-package-dir.d.ts +9 -2
- package/dist/expo/plugins/resolve-sdk-package-dir.js +38 -11
- package/dist/expo/plugins/withDeviceInfo.js +2 -1
- package/dist/expo/plugins/withMobileBundle.d.ts +5 -4
- package/dist/expo/plugins/withMobileBundle.js +42 -58
- package/dist/index.d.ts +6 -2
- package/dist/index.js +23 -2
- package/dist/models/registry/models.d.ts +3725 -514
- package/dist/models/registry/models.js +7269 -3910
- package/dist/models/update-models/naming.js +22 -8
- package/dist/pear/pre.d.ts +3 -0
- package/dist/pear/pre.js +9 -2
- package/dist/schemas/cancel.d.ts +172 -28
- package/dist/schemas/cancel.js +120 -19
- package/dist/schemas/classification.d.ts +53 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +860 -1379
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +17 -0
- package/dist/schemas/completion-event.js +12 -1
- package/dist/schemas/completion-stream.d.ts +16 -1
- package/dist/schemas/completion-stream.js +25 -7
- package/dist/schemas/download-asset.d.ts +25 -9
- package/dist/schemas/download-asset.js +17 -6
- package/dist/schemas/embed.d.ts +1 -0
- package/dist/schemas/embed.js +5 -0
- package/dist/schemas/engine-addon-map.d.ts +3 -0
- package/dist/schemas/engine-addon-map.js +18 -5
- package/dist/schemas/error.d.ts +31 -0
- package/dist/schemas/error.js +29 -1
- package/dist/schemas/finetune.d.ts +2 -0
- package/dist/schemas/finetune.js +5 -0
- package/dist/schemas/get-model-info.d.ts +10 -6
- package/dist/schemas/get-model-info.js +2 -0
- package/dist/schemas/index.d.ts +3 -0
- package/dist/schemas/index.js +3 -0
- package/dist/schemas/llamacpp-config.d.ts +64 -8
- package/dist/schemas/llamacpp-config.js +12 -0
- package/dist/schemas/load-model.d.ts +4644 -8605
- package/dist/schemas/load-model.js +90 -7
- package/dist/schemas/model-src-utils.d.ts +34 -14
- package/dist/schemas/model-types.d.ts +54 -14
- package/dist/schemas/model-types.js +28 -5
- package/dist/schemas/ocr.d.ts +8 -3
- package/dist/schemas/plugin.d.ts +60 -6
- package/dist/schemas/plugin.js +31 -5
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +17 -9
- package/dist/schemas/registry.d.ts +74 -30
- package/dist/schemas/registry.js +5 -0
- package/dist/schemas/reload-config.d.ts +34 -14
- package/dist/schemas/sdcpp-config.d.ts +445 -16
- package/dist/schemas/sdcpp-config.js +387 -26
- package/dist/schemas/sdk-config.d.ts +2431 -2723
- package/dist/schemas/sdk-config.js +16 -1
- package/dist/schemas/sdk-errors-client.d.ts +6 -0
- package/dist/schemas/sdk-errors-client.js +31 -1
- package/dist/schemas/sdk-errors-server.d.ts +8 -1
- package/dist/schemas/sdk-errors-server.js +45 -5
- package/dist/schemas/text-to-speech.d.ts +81 -698
- package/dist/schemas/text-to-speech.js +46 -19
- package/dist/schemas/transcription-config.d.ts +40 -229
- package/dist/schemas/transcription-config.js +76 -15
- package/dist/schemas/transcription.d.ts +108 -2
- package/dist/schemas/transcription.js +81 -0
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +13 -2
- package/dist/schemas/translation-config.d.ts +90 -40
- package/dist/schemas/unload-model.d.ts +2 -1
- package/dist/schemas/unload-model.js +4 -1
- package/dist/schemas/vla.d.ts +87 -0
- package/dist/schemas/vla.js +108 -0
- package/dist/scripts/run-unit-tests.js +22 -3
- package/dist/server/bare/ops/cancel.d.ts +27 -1
- package/dist/server/bare/ops/cancel.js +36 -10
- package/dist/server/bare/ops/embed.d.ts +3 -3
- package/dist/server/bare/ops/embed.js +144 -28
- package/dist/server/bare/ops/kv-cache-utils.d.ts +0 -7
- package/dist/server/bare/ops/kv-cache-utils.js +8 -47
- package/dist/server/bare/ops/transcribe.d.ts +11 -5
- package/dist/server/bare/ops/transcribe.js +230 -50
- package/dist/server/bare/ops/translate.d.ts +1 -1
- package/dist/server/bare/ops/translate.js +197 -88
- package/dist/server/bare/plugins/ggml-classification/ops/classify.d.ts +6 -0
- package/dist/server/bare/plugins/ggml-classification/ops/classify.js +30 -0
- package/dist/server/bare/plugins/ggml-classification/plugin.d.ts +32 -0
- package/dist/server/bare/plugins/ggml-classification/plugin.js +55 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-hparams.d.ts +3 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-run.d.ts +3 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-run.js +75 -0
- package/dist/server/bare/plugins/ggml-vla/plugin.d.ts +54 -0
- package/dist/server/bare/plugins/ggml-vla/plugin.js +57 -0
- package/dist/server/bare/plugins/index.d.ts +3 -1
- package/dist/server/bare/plugins/index.js +3 -1
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts +12 -3
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +173 -172
- package/dist/server/bare/plugins/llamacpp-completion/ops/context-overflow.d.ts +29 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/finetune.js +125 -21
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.d.ts +154 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.js +448 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.d.ts +21 -73
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.js +18 -101
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +36 -3
- package/dist/server/bare/plugins/llamacpp-completion/plugin.js +183 -85
- package/dist/server/bare/plugins/llamacpp-completion/transform.d.ts +10 -0
- package/dist/server/bare/plugins/llamacpp-completion/transform.js +34 -0
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.d.ts +7 -0
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +10 -1
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +42 -15
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +5 -1
- package/dist/server/bare/plugins/onnx-ocr/plugin.d.ts +8 -3
- package/dist/server/bare/plugins/onnx-ocr/plugin.js +3 -0
- package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +73 -244
- package/dist/server/bare/plugins/parakeet-transcription/plugin.js +68 -96
- package/dist/server/bare/plugins/sdcpp-generation/ops/diffusion.js +8 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/upscale.d.ts +3 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/upscale.js +40 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/video.d.ts +3 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/video.js +199 -0
- package/dist/server/bare/plugins/sdcpp-generation/plugin.d.ts +288 -15
- package/dist/server/bare/plugins/sdcpp-generation/plugin.js +150 -7
- package/dist/server/bare/plugins/{onnx-tts → tts-ggml}/ops/text-to-speech-stream.js +2 -15
- package/dist/server/bare/plugins/{onnx-tts → tts-ggml}/ops/text-to-speech.js +2 -1
- package/dist/server/bare/plugins/tts-ggml/plugin.d.ts +75 -0
- package/dist/server/bare/plugins/tts-ggml/plugin.js +189 -0
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +47 -3
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.js +61 -14
- package/dist/server/bare/rag-hyperdb/index.d.ts +1 -1
- package/dist/server/bare/rag-hyperdb/index.js +1 -1
- package/dist/server/bare/rag-hyperdb/rag-operation-manager.d.ts +26 -4
- package/dist/server/bare/rag-hyperdb/rag-operation-manager.js +65 -28
- package/dist/server/bare/registry/model-config-utils.js +7 -0
- package/dist/server/bare/runtime/disposable-scope.d.ts +36 -0
- package/dist/server/bare/runtime/disposable-scope.js +69 -0
- package/dist/server/bare/runtime/index.d.ts +8 -0
- package/dist/server/bare/runtime/index.js +3 -0
- package/dist/server/bare/runtime/request-context.d.ts +49 -0
- package/dist/server/bare/runtime/request-context.js +1 -0
- package/dist/server/bare/runtime/request-id.d.ts +12 -0
- package/dist/server/bare/runtime/request-id.js +14 -0
- package/dist/server/bare/runtime/request-registry-singleton.d.ts +4 -0
- package/dist/server/bare/runtime/request-registry-singleton.js +27 -0
- package/dist/server/bare/runtime/request-registry.d.ts +136 -0
- package/dist/server/bare/runtime/request-registry.js +297 -0
- package/dist/server/bare/runtime/with-request-context.d.ts +22 -0
- package/dist/server/bare/runtime/with-request-context.js +53 -0
- package/dist/server/bare/runtime-lifecycle.js +9 -9
- package/dist/server/bare/types/addon-responses.d.ts +12 -1
- package/dist/server/env.js +4 -1
- package/dist/server/plugins/index.d.ts +1 -1
- package/dist/server/plugins/index.js +1 -1
- package/dist/server/plugins/registry.d.ts +1 -0
- package/dist/server/plugins/registry.js +5 -0
- package/dist/server/rpc/handler-registry.js +30 -7
- package/dist/server/rpc/handler-selection.d.ts +9 -0
- package/dist/server/rpc/handler-selection.js +14 -0
- package/dist/server/rpc/handler-utils.js +5 -13
- package/dist/server/rpc/handlers/cancel-delegated.js +28 -31
- package/dist/server/rpc/handlers/cancelHandler.d.ts +30 -1
- package/dist/server/rpc/handlers/cancelHandler.js +54 -22
- package/dist/server/rpc/handlers/classify.d.ts +3 -0
- package/dist/server/rpc/handlers/classify.js +4 -0
- package/dist/server/rpc/handlers/delete-cache.d.ts +9 -0
- package/dist/server/rpc/handlers/delete-cache.js +13 -15
- package/dist/server/rpc/handlers/download-asset.js +133 -40
- package/dist/server/rpc/handlers/index.d.ts +4 -0
- package/dist/server/rpc/handlers/index.js +4 -0
- package/dist/server/rpc/handlers/load-model/download-manager.d.ts +60 -2
- package/dist/server/rpc/handlers/load-model/download-manager.js +141 -8
- package/dist/server/rpc/handlers/load-model/handler.js +216 -98
- package/dist/server/rpc/handlers/load-model/http.js +3 -3
- package/dist/server/rpc/handlers/load-model/hyperdrive.js +1 -1
- package/dist/server/rpc/handlers/load-model/registry.js +1 -1
- package/dist/server/rpc/handlers/load-model/resolve-session.d.ts +20 -1
- package/dist/server/rpc/handlers/load-model/resolve-session.js +6 -5
- package/dist/server/rpc/handlers/load-model/resolve.d.ts +3 -2
- package/dist/server/rpc/handlers/load-model/resolve.js +18 -6
- package/dist/server/rpc/handlers/load-model/types.d.ts +24 -0
- package/dist/server/rpc/handlers/rag.js +133 -15
- package/dist/server/rpc/handlers/upscale-stream.d.ts +3 -0
- package/dist/server/rpc/handlers/upscale-stream.js +4 -0
- package/dist/server/rpc/handlers/video-stream.d.ts +3 -0
- package/dist/server/rpc/handlers/video-stream.js +4 -0
- package/dist/server/rpc/profiling/operation-metrics.js +23 -0
- package/dist/server/utils/completion-normalizer.js +8 -0
- package/dist/server/utils/config-transform.js +1 -1
- package/dist/server/utils/tools/dialect.js +4 -0
- package/dist/server/utils/tools/parser.js +16 -1
- package/dist/server/utils/tools/parsers/gemma4native.d.ts +4 -0
- package/dist/server/utils/tools/parsers/gemma4native.js +60 -0
- package/dist/server/utils/tools/parsers/qwen35.d.ts +4 -0
- package/dist/server/utils/tools/parsers/qwen35.js +120 -0
- package/dist/server/worker.js +14 -11
- package/dist/utils/aggregate-events.d.ts +9 -0
- package/dist/utils/aggregate-events.js +18 -3
- package/dist/utils/decorate-promise.d.ts +34 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/errors-client.d.ts +18 -0
- package/dist/utils/errors-client.js +31 -0
- package/dist/utils/errors-server.d.ts +115 -2
- package/dist/utils/errors-server.js +148 -3
- package/package.json +68 -77
- package/dist/server/bare/plugins/onnx-tts/plugin.d.ts +0 -490
- package/dist/server/bare/plugins/onnx-tts/plugin.js +0 -266
- package/dist/server/bare/plugins/onnx-tts/wav-helper.d.ts +0 -14
- package/dist/server/bare/plugins/onnx-tts/wav-helper.js +0 -107
- /package/dist/server/bare/plugins/{onnx-tts → tts-ggml}/ops/text-to-speech-stream.d.ts +0 -0
- /package/dist/server/bare/plugins/{onnx-tts → tts-ggml}/ops/text-to-speech.d.ts +0 -0
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
import { AbortController } from "bare-abort-controller";
|
|
2
|
+
import { createDisposableScope, } from "./disposable-scope.js";
|
|
3
|
+
import { RequestIdConflictError, RequestRejectedByPolicyError, } from "../../../utils/errors-server.js";
|
|
4
|
+
import { getServerLogger } from "../../../logging/index.js";
|
|
5
|
+
/**
|
|
6
|
+
* Tuning knobs for the "cancelled-before-begin" bookkeeping set.
|
|
7
|
+
*
|
|
8
|
+
* The race window is bounded by the client-to-server round-trip: a
|
|
9
|
+
* `cancel({ requestId })` issued by the client at the same time as the
|
|
10
|
+
* matching `completion(...)` either lands first (and we need to remember
|
|
11
|
+
* it long enough for the server's `begin(...)` to follow) or lands
|
|
12
|
+
* second (and we never touch this set). 30 seconds is overkill for a
|
|
13
|
+
* 500ms round-trip but gives slow networks / pause-the-debugger
|
|
14
|
+
* scenarios enough slack while still bounding worst-case retention.
|
|
15
|
+
*
|
|
16
|
+
* The size cap protects against a buggy or malicious client firing a
|
|
17
|
+
* stream of cancels for ids that never get a `begin(...)` follow-up —
|
|
18
|
+
* each `cancel({ requestId })` that doesn't match an in-flight context
|
|
19
|
+
* inserts one entry, so without a cap the worker would grow the map
|
|
20
|
+
* unbounded. At the cap, the oldest entry is evicted.
|
|
21
|
+
*
|
|
22
|
+
* Tweak with care: both bounds appear in the registry race test
|
|
23
|
+
* (`bounded cancel-before-begin set does not grow past its cap`).
|
|
24
|
+
*/
|
|
25
|
+
const CANCEL_BEFORE_BEGIN_MAX_ENTRIES = 128;
|
|
26
|
+
const CANCEL_BEFORE_BEGIN_TTL_MS = 30_000;
|
|
27
|
+
export function createRequestRegistry(options) {
|
|
28
|
+
const entries = new Map();
|
|
29
|
+
const policies = new Map();
|
|
30
|
+
const logger = options?.logger ?? getServerLogger();
|
|
31
|
+
function logLifecycle(event, ctx, durationMs) {
|
|
32
|
+
const modelId = ctx.modelId !== undefined ? ctx.modelId : "-";
|
|
33
|
+
const base = `[request-lifecycle] ${event} requestId=${ctx.requestId} kind=${ctx.kind} modelId=${modelId} state=${ctx.state}`;
|
|
34
|
+
const line = durationMs !== undefined ? `${base} durationMs=${durationMs}` : base;
|
|
35
|
+
// `failed` end emits at `warn` so log shippers can alert on
|
|
36
|
+
// `level>=warn` for this prefix without parsing `state=failed`
|
|
37
|
+
// out of the message body. Everything else stays at `info`.
|
|
38
|
+
if (event === "end" && ctx.state === "failed") {
|
|
39
|
+
logger.warn(line);
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
logger.info(line);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* "Cancelled-before-begin" tripwire. A
|
|
47
|
+
* `cancel({ requestId })` whose target isn't yet in `entries` records
|
|
48
|
+
* the id here; the subsequent `begin({ requestId: <same id> })` then
|
|
49
|
+
* aborts the new controller before returning. Map order is insertion
|
|
50
|
+
* order — the iterator's first key is the oldest entry, which makes
|
|
51
|
+
* the size-cap eviction free.
|
|
52
|
+
*
|
|
53
|
+
* Invariants:
|
|
54
|
+
* - Every read path (`begin`, `cancel`-by-id) calls
|
|
55
|
+
* `pruneCancelBeforeBeginExpired()` first so a 30s+ stale entry
|
|
56
|
+
* never decides a fresh `begin(...)`.
|
|
57
|
+
* - Insertion enforces the size cap by evicting the oldest entry
|
|
58
|
+
* when at capacity — a malicious client cannot grow this map
|
|
59
|
+
* unbounded.
|
|
60
|
+
* - On a successful `begin(...)` match, the entry is consumed
|
|
61
|
+
* (removed) so a second `begin(...)` with the same id (which
|
|
62
|
+
* would itself throw `RequestIdConflictError`) doesn't see a
|
|
63
|
+
* phantom pre-cancel.
|
|
64
|
+
*/
|
|
65
|
+
const cancelledBeforeBegin = new Map();
|
|
66
|
+
function pruneCancelBeforeBeginExpired(now = Date.now()) {
|
|
67
|
+
if (cancelledBeforeBegin.size === 0)
|
|
68
|
+
return;
|
|
69
|
+
const cutoff = now - CANCEL_BEFORE_BEGIN_TTL_MS;
|
|
70
|
+
for (const [id, entry] of cancelledBeforeBegin) {
|
|
71
|
+
if (entry.at > cutoff)
|
|
72
|
+
break; // Insertion order ⇒ rest are newer.
|
|
73
|
+
cancelledBeforeBegin.delete(id);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
function recordCancelBeforeBegin(requestId, reason) {
|
|
77
|
+
const now = Date.now();
|
|
78
|
+
pruneCancelBeforeBeginExpired(now);
|
|
79
|
+
// Re-canceling an id that is already tracked refreshes its TTL but
|
|
80
|
+
// keeps the original reason — the first cancel "won" the race.
|
|
81
|
+
if (cancelledBeforeBegin.has(requestId)) {
|
|
82
|
+
const existing = cancelledBeforeBegin.get(requestId);
|
|
83
|
+
cancelledBeforeBegin.delete(requestId);
|
|
84
|
+
cancelledBeforeBegin.set(requestId, { ...existing, at: now });
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
if (cancelledBeforeBegin.size >= CANCEL_BEFORE_BEGIN_MAX_ENTRIES) {
|
|
88
|
+
const oldest = cancelledBeforeBegin.keys().next().value;
|
|
89
|
+
if (oldest !== undefined)
|
|
90
|
+
cancelledBeforeBegin.delete(oldest);
|
|
91
|
+
}
|
|
92
|
+
cancelledBeforeBegin.set(requestId, reason !== undefined ? { at: now, reason } : { at: now });
|
|
93
|
+
}
|
|
94
|
+
function consumeCancelBeforeBegin(requestId) {
|
|
95
|
+
pruneCancelBeforeBeginExpired();
|
|
96
|
+
const entry = cancelledBeforeBegin.get(requestId);
|
|
97
|
+
if (!entry)
|
|
98
|
+
return undefined;
|
|
99
|
+
cancelledBeforeBegin.delete(requestId);
|
|
100
|
+
return entry;
|
|
101
|
+
}
|
|
102
|
+
function cancelEntry(entry, reason) {
|
|
103
|
+
if (entry.controller.signal.aborted)
|
|
104
|
+
return false;
|
|
105
|
+
entry.ctx.state = "cancelling";
|
|
106
|
+
entry.controller.abort(reason);
|
|
107
|
+
logLifecycle("cancel", entry.ctx);
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
async function disposeEntry(entry, outcome) {
|
|
111
|
+
if (entry.scope.disposed)
|
|
112
|
+
return;
|
|
113
|
+
entry.ctx.state = outcome;
|
|
114
|
+
entry.detachParent();
|
|
115
|
+
logLifecycle("end", entry.ctx, Date.now() - entry.startedAt);
|
|
116
|
+
// Pull the entry out before unwinding so observers (e.g. a `cancel(...)`
|
|
117
|
+
// racing with dispose) don't see a half-disposed context.
|
|
118
|
+
entries.delete(entry.ctx.requestId);
|
|
119
|
+
await entry.scope[Symbol.asyncDispose]();
|
|
120
|
+
}
|
|
121
|
+
function begin(opts) {
|
|
122
|
+
if (entries.has(opts.requestId)) {
|
|
123
|
+
throw new RequestIdConflictError(opts.requestId);
|
|
124
|
+
}
|
|
125
|
+
// Admission control runs before allocation so a rejected begin
|
|
126
|
+
// leaves no controller / scope behind.
|
|
127
|
+
const policy = policies.get(opts.kind);
|
|
128
|
+
if (policy && policy.oneAtATimePerModel && opts.modelId !== undefined) {
|
|
129
|
+
for (const existing of entries.values()) {
|
|
130
|
+
if (existing.ctx.kind !== opts.kind)
|
|
131
|
+
continue;
|
|
132
|
+
if (existing.ctx.modelId !== opts.modelId)
|
|
133
|
+
continue;
|
|
134
|
+
throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, opts.modelId, `another ${opts.kind} request (${existing.ctx.requestId}) is already running on this model`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
const controller = new AbortController();
|
|
138
|
+
const scope = createDisposableScope();
|
|
139
|
+
// Stop-button race close. If a
|
|
140
|
+
// `cancel({ requestId })` already arrived for this id, abort the
|
|
141
|
+
// new controller before observers can subscribe to it. The
|
|
142
|
+
// tripwire entry is consumed so a later, separate `begin(...)`
|
|
143
|
+
// with the same id is unaffected (in practice ids are UUIDv4 and
|
|
144
|
+
// never reused; this guard just keeps the contract self-
|
|
145
|
+
// consistent under retries).
|
|
146
|
+
const preCancel = consumeCancelBeforeBegin(opts.requestId);
|
|
147
|
+
if (preCancel) {
|
|
148
|
+
controller.abort(preCancel.reason);
|
|
149
|
+
}
|
|
150
|
+
let detachParent = () => { };
|
|
151
|
+
if (opts.parentSignal) {
|
|
152
|
+
const parent = opts.parentSignal;
|
|
153
|
+
if (parent.aborted) {
|
|
154
|
+
controller.abort(parent.reason);
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
const onParentAbort = () => controller.abort(parent.reason);
|
|
158
|
+
parent.addEventListener("abort", onParentAbort, { once: true });
|
|
159
|
+
detachParent = () => parent.removeEventListener("abort", onParentAbort);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
const ctx = {
|
|
163
|
+
requestId: opts.requestId,
|
|
164
|
+
kind: opts.kind,
|
|
165
|
+
modelId: opts.modelId,
|
|
166
|
+
signal: controller.signal,
|
|
167
|
+
scope,
|
|
168
|
+
// Land the context in `cancelling` from the outset whenever the
|
|
169
|
+
// controller was already aborted by `begin(...)` itself — either
|
|
170
|
+
// the Stop-button race (`preCancel`) or a `parentSignal` that was
|
|
171
|
+
// already aborted at begin time. Both branches abort the
|
|
172
|
+
// controller above, so without this guard observers would see a
|
|
173
|
+
// momentarily-`running` context with an already-aborted signal.
|
|
174
|
+
state: preCancel || opts.parentSignal?.aborted ? "cancelling" : "running",
|
|
175
|
+
};
|
|
176
|
+
const entry = {
|
|
177
|
+
ctx,
|
|
178
|
+
controller,
|
|
179
|
+
scope,
|
|
180
|
+
detachParent,
|
|
181
|
+
startedAt: Date.now(),
|
|
182
|
+
};
|
|
183
|
+
entries.set(opts.requestId, entry);
|
|
184
|
+
logLifecycle("begin", ctx);
|
|
185
|
+
return {
|
|
186
|
+
get requestId() {
|
|
187
|
+
return ctx.requestId;
|
|
188
|
+
},
|
|
189
|
+
get kind() {
|
|
190
|
+
return ctx.kind;
|
|
191
|
+
},
|
|
192
|
+
get modelId() {
|
|
193
|
+
return ctx.modelId;
|
|
194
|
+
},
|
|
195
|
+
get signal() {
|
|
196
|
+
return ctx.signal;
|
|
197
|
+
},
|
|
198
|
+
get scope() {
|
|
199
|
+
return ctx.scope;
|
|
200
|
+
},
|
|
201
|
+
get state() {
|
|
202
|
+
return ctx.state;
|
|
203
|
+
},
|
|
204
|
+
set state(next) {
|
|
205
|
+
ctx.state = next;
|
|
206
|
+
},
|
|
207
|
+
[Symbol.asyncDispose]: async () => {
|
|
208
|
+
await disposeEntry(entry, derivedTerminalState(ctx));
|
|
209
|
+
},
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
function get(requestId) {
|
|
213
|
+
return entries.get(requestId)?.ctx ?? null;
|
|
214
|
+
}
|
|
215
|
+
function list() {
|
|
216
|
+
return Array.from(entries.values(), (e) => e.ctx);
|
|
217
|
+
}
|
|
218
|
+
function cancel(target) {
|
|
219
|
+
let cancelled = 0;
|
|
220
|
+
if ("requestId" in target) {
|
|
221
|
+
const entry = entries.get(target.requestId);
|
|
222
|
+
if (entry) {
|
|
223
|
+
if (cancelEntry(entry, target.reason))
|
|
224
|
+
cancelled++;
|
|
225
|
+
return cancelled;
|
|
226
|
+
}
|
|
227
|
+
// Stop-button race: the client beat its own
|
|
228
|
+
// `begin(...)`. Record the cancel so the next matching `begin`
|
|
229
|
+
// aborts immediately. The return value stays 0 — no in-flight
|
|
230
|
+
// request was matched, which is still the truth — but the
|
|
231
|
+
// *effective* cancel will land when the begin arrives.
|
|
232
|
+
recordCancelBeforeBegin(target.requestId, target.reason);
|
|
233
|
+
return cancelled;
|
|
234
|
+
}
|
|
235
|
+
for (const entry of entries.values()) {
|
|
236
|
+
if (entry.ctx.modelId !== target.modelId)
|
|
237
|
+
continue;
|
|
238
|
+
if (target.kind && entry.ctx.kind !== target.kind)
|
|
239
|
+
continue;
|
|
240
|
+
if (cancelEntry(entry, target.reason))
|
|
241
|
+
cancelled++;
|
|
242
|
+
}
|
|
243
|
+
return cancelled;
|
|
244
|
+
}
|
|
245
|
+
function cancelAll(reason) {
|
|
246
|
+
for (const entry of entries.values()) {
|
|
247
|
+
cancelEntry(entry, reason);
|
|
248
|
+
}
|
|
249
|
+
// The interface returns Promise<void> so we can later make this an
|
|
250
|
+
// async sweep that awaits per-handler scope unwinding (e.g. join on
|
|
251
|
+
// the disposers). Today every handler unwinds on its own dispose
|
|
252
|
+
// path, so the function only needs to fire-and-forget the abort.
|
|
253
|
+
return Promise.resolve();
|
|
254
|
+
}
|
|
255
|
+
async function end(requestId, outcome) {
|
|
256
|
+
const entry = entries.get(requestId);
|
|
257
|
+
if (!entry)
|
|
258
|
+
return;
|
|
259
|
+
await disposeEntry(entry, outcome);
|
|
260
|
+
}
|
|
261
|
+
function policy(opts) {
|
|
262
|
+
policies.set(opts.kind, opts);
|
|
263
|
+
}
|
|
264
|
+
return {
|
|
265
|
+
begin,
|
|
266
|
+
get,
|
|
267
|
+
list,
|
|
268
|
+
cancel,
|
|
269
|
+
cancelAll,
|
|
270
|
+
end,
|
|
271
|
+
policy,
|
|
272
|
+
// Test-only: lets the registry race tests assert the bound
|
|
273
|
+
// invariants on the internal "cancelled-before-begin" set without
|
|
274
|
+
// exposing it as a public surface. Kept off the `RequestRegistry`
|
|
275
|
+
// interface (typed via the augmented return type below) so handler
|
|
276
|
+
// code can't depend on it accidentally.
|
|
277
|
+
__cancelBeforeBeginSize: () => cancelledBeforeBegin.size,
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Test-only knobs exported for `request-registry.test.ts` so the bound
|
|
282
|
+
* assertions can pin the documented limits without re-reading them via
|
|
283
|
+
* fragile string comparison. **Not part of the public SDK surface.**
|
|
284
|
+
*
|
|
285
|
+
* @internal
|
|
286
|
+
*/
|
|
287
|
+
export const __requestRegistryTestHooks = {
|
|
288
|
+
cancelBeforeBeginMaxEntries: CANCEL_BEFORE_BEGIN_MAX_ENTRIES,
|
|
289
|
+
cancelBeforeBeginTtlMs: CANCEL_BEFORE_BEGIN_TTL_MS,
|
|
290
|
+
};
|
|
291
|
+
function derivedTerminalState(ctx) {
|
|
292
|
+
if (ctx.state === "failed")
|
|
293
|
+
return "failed";
|
|
294
|
+
if (ctx.signal.aborted || ctx.state === "cancelled")
|
|
295
|
+
return "cancelled";
|
|
296
|
+
return "completed";
|
|
297
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Logger } from "../../../logging/types";
|
|
2
|
+
export interface RequestLogContext {
|
|
3
|
+
requestId: string;
|
|
4
|
+
kind: string;
|
|
5
|
+
modelId: string | undefined;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Wraps `logger` so every emit is prefixed with
|
|
9
|
+
* `[request-lifecycle <kind> requestId=<id> modelId=<modelId>] `
|
|
10
|
+
* (the `modelId=...` segment is dropped when absent). The wrapper is a
|
|
11
|
+
* thin shim: `setLevel` / `getLevel` / `addTransport` / `setConsoleOutput`
|
|
12
|
+
* pass through to the underlying logger, and transport callbacks receive
|
|
13
|
+
* the prefixed message.
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* await using ctx = registry.begin({ requestId, kind: "completion", modelId });
|
|
17
|
+
* const log = withRequestContext(getServerLogger(), ctx);
|
|
18
|
+
* log.info("decoding token 7");
|
|
19
|
+
* // → "[request-lifecycle completion requestId=<id> modelId=<id>] decoding token 7"
|
|
20
|
+
*/
|
|
21
|
+
export declare function withRequestContext(logger: Logger, ctx: RequestLogContext): Logger;
|
|
22
|
+
//# sourceMappingURL=with-request-context.d.ts.map
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wraps `logger` so every emit is prefixed with
|
|
3
|
+
* `[request-lifecycle <kind> requestId=<id> modelId=<modelId>] `
|
|
4
|
+
* (the `modelId=...` segment is dropped when absent). The wrapper is a
|
|
5
|
+
* thin shim: `setLevel` / `getLevel` / `addTransport` / `setConsoleOutput`
|
|
6
|
+
* pass through to the underlying logger, and transport callbacks receive
|
|
7
|
+
* the prefixed message.
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* await using ctx = registry.begin({ requestId, kind: "completion", modelId });
|
|
11
|
+
* const log = withRequestContext(getServerLogger(), ctx);
|
|
12
|
+
* log.info("decoding token 7");
|
|
13
|
+
* // → "[request-lifecycle completion requestId=<id> modelId=<id>] decoding token 7"
|
|
14
|
+
*/
|
|
15
|
+
export function withRequestContext(logger, ctx) {
|
|
16
|
+
const prefix = ctx.modelId !== undefined
|
|
17
|
+
? `[request-lifecycle ${ctx.kind} requestId=${ctx.requestId} modelId=${ctx.modelId}] `
|
|
18
|
+
: `[request-lifecycle ${ctx.kind} requestId=${ctx.requestId}] `;
|
|
19
|
+
function pick(method) {
|
|
20
|
+
switch (method) {
|
|
21
|
+
case "error":
|
|
22
|
+
return logger.error;
|
|
23
|
+
case "warn":
|
|
24
|
+
return logger.warn;
|
|
25
|
+
case "info":
|
|
26
|
+
return logger.info;
|
|
27
|
+
case "debug":
|
|
28
|
+
return logger.debug;
|
|
29
|
+
case "trace":
|
|
30
|
+
return logger.trace;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function emit(method, args) {
|
|
34
|
+
const sink = pick(method);
|
|
35
|
+
if (args.length === 0) {
|
|
36
|
+
sink(prefix);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const [first, ...rest] = args;
|
|
40
|
+
sink(prefix + String(first), ...rest);
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
error: (...args) => emit("error", args),
|
|
44
|
+
warn: (...args) => emit("warn", args),
|
|
45
|
+
info: (...args) => emit("info", args),
|
|
46
|
+
debug: (...args) => emit("debug", args),
|
|
47
|
+
trace: (...args) => emit("trace", args),
|
|
48
|
+
setLevel: (level) => logger.setLevel(level),
|
|
49
|
+
getLevel: () => logger.getLevel(),
|
|
50
|
+
addTransport: (transport) => logger.addTransport(transport),
|
|
51
|
+
setConsoleOutput: (enabled) => logger.setConsoleOutput(enabled),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -78,17 +78,17 @@ export async function suspendRuntime() {
|
|
|
78
78
|
return suspendRuntime();
|
|
79
79
|
}
|
|
80
80
|
state = "suspending";
|
|
81
|
-
logger.info(`⏸️ Suspending runtime (${
|
|
81
|
+
logger.info(`⏸️ Suspending runtime (${stores.size} stores, ${swarms.size} swarms)`);
|
|
82
82
|
transitionPromise = (async () => {
|
|
83
83
|
const swarmSnapshot = Array.from(swarms.entries());
|
|
84
84
|
const storeSnapshot = Array.from(stores.entries());
|
|
85
|
+
await runPhase(storeSnapshot, stores, async (store) => {
|
|
86
|
+
await store.suspend();
|
|
87
|
+
}, "suspend-store");
|
|
85
88
|
await runPhase(swarmSnapshot, swarms, async (swarm) => {
|
|
86
89
|
if (!swarm.suspended)
|
|
87
90
|
await swarm.suspend();
|
|
88
91
|
}, "suspend-swarm");
|
|
89
|
-
await runPhase(storeSnapshot, stores, async (store) => {
|
|
90
|
-
await store.suspend();
|
|
91
|
-
}, "suspend-store");
|
|
92
92
|
})()
|
|
93
93
|
.then(() => {
|
|
94
94
|
state = "suspended";
|
|
@@ -116,16 +116,16 @@ export async function resumeRuntime() {
|
|
|
116
116
|
return resumeRuntime();
|
|
117
117
|
}
|
|
118
118
|
state = "resuming";
|
|
119
|
-
logger.info(`▶️ Resuming runtime (${
|
|
119
|
+
logger.info(`▶️ Resuming runtime (${swarms.size} swarms, ${stores.size} stores)`);
|
|
120
120
|
transitionPromise = (async () => {
|
|
121
|
-
const storeSnapshot = Array.from(stores.entries());
|
|
122
121
|
const swarmSnapshot = Array.from(swarms.entries());
|
|
123
|
-
|
|
124
|
-
await store.resume();
|
|
125
|
-
}, "resume-store");
|
|
122
|
+
const storeSnapshot = Array.from(stores.entries());
|
|
126
123
|
await runPhase(swarmSnapshot, swarms, async (swarm) => {
|
|
127
124
|
await swarm.resume();
|
|
128
125
|
}, "resume-swarm");
|
|
126
|
+
await runPhase(storeSnapshot, stores, async (store) => {
|
|
127
|
+
await store.resume();
|
|
128
|
+
}, "resume-store");
|
|
129
129
|
})()
|
|
130
130
|
.then(() => {
|
|
131
131
|
state = "active";
|
|
@@ -2,6 +2,7 @@ export interface LlmStats {
|
|
|
2
2
|
TTFT?: number;
|
|
3
3
|
TPS?: number;
|
|
4
4
|
CacheTokens?: number;
|
|
5
|
+
promptTokens?: number;
|
|
5
6
|
generatedTokens?: number;
|
|
6
7
|
backendDevice?: "cpu" | "gpu";
|
|
7
8
|
}
|
|
@@ -60,8 +61,18 @@ export interface TranscribeAddonSegment {
|
|
|
60
61
|
toAppend?: boolean;
|
|
61
62
|
id?: number;
|
|
62
63
|
}
|
|
64
|
+
export interface TranscribeAddonVadEvent {
|
|
65
|
+
type: "vad";
|
|
66
|
+
speaking: boolean;
|
|
67
|
+
probability: number;
|
|
68
|
+
}
|
|
69
|
+
export interface TranscribeAddonEndOfTurnEvent {
|
|
70
|
+
type: "endOfTurn";
|
|
71
|
+
silenceDurationMs: number;
|
|
72
|
+
}
|
|
73
|
+
export type TranscribeAddonOutput = Array<TranscribeAddonSegment> | TranscribeAddonVadEvent | TranscribeAddonEndOfTurnEvent;
|
|
63
74
|
export interface TranscribeResponse {
|
|
64
75
|
stats?: TranscribeStats;
|
|
65
|
-
iterate(): AsyncIterable<
|
|
76
|
+
iterate(): AsyncIterable<TranscribeAddonOutput>;
|
|
66
77
|
}
|
|
67
78
|
//# sourceMappingURL=addon-responses.d.ts.map
|
package/dist/server/env.js
CHANGED
|
@@ -11,7 +11,10 @@ let validatedEnv = null;
|
|
|
11
11
|
* Returns whether RPC config was parsed from arguments.
|
|
12
12
|
*/
|
|
13
13
|
export function initEnv() {
|
|
14
|
-
const defaultHomeDir =
|
|
14
|
+
const defaultHomeDir =
|
|
15
|
+
// Snap's HOME can be revision-scoped; SNAP_USER_COMMON is stable.
|
|
16
|
+
process.env["SNAP_USER_COMMON"] ??
|
|
17
|
+
process.env["HOME"] ??
|
|
15
18
|
process.env["USERPROFILE"] ??
|
|
16
19
|
"/tmp";
|
|
17
20
|
let envConfig = {
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { registerPlugin, getPlugin, getPluginHandler, hasPlugin, getAllPlugins, clearPlugins, unregisterPlugin, } from "./registry";
|
|
1
|
+
export { registerPlugin, registerPlugins, getPlugin, getPluginHandler, hasPlugin, getAllPlugins, clearPlugins, unregisterPlugin, } from "./registry";
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export { registerPlugin, getPlugin, getPluginHandler, hasPlugin, getAllPlugins, clearPlugins, unregisterPlugin, } from "./registry.js";
|
|
1
|
+
export { registerPlugin, registerPlugins, getPlugin, getPluginHandler, hasPlugin, getAllPlugins, clearPlugins, unregisterPlugin, } from "./registry.js";
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type QvacPlugin, type PluginHandlerDefinition } from "../../schemas/plugin";
|
|
2
2
|
export declare function registerPlugin(plugin: QvacPlugin): void;
|
|
3
|
+
export declare function registerPlugins(pluginList: readonly QvacPlugin[]): void;
|
|
3
4
|
export declare function getPlugin(modelType: string): QvacPlugin | undefined;
|
|
4
5
|
export declare function getPluginHandler(modelType: string, handlerName: string): PluginHandlerDefinition | undefined;
|
|
5
6
|
export declare function hasPlugin(modelType: string): boolean;
|
|
@@ -43,6 +43,11 @@ export function registerPlugin(plugin) {
|
|
|
43
43
|
loggingModule.setLogger(createAddonLoggerCallback(plugin.logging.namespace));
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
|
+
export function registerPlugins(pluginList) {
|
|
47
|
+
for (const plugin of pluginList) {
|
|
48
|
+
registerPlugin(plugin);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
46
51
|
export function getPlugin(modelType) {
|
|
47
52
|
return plugins.get(modelType);
|
|
48
53
|
}
|
|
@@ -27,6 +27,9 @@ import { handleFinetune } from "./handlers/finetune.js";
|
|
|
27
27
|
import { handleHeartbeatDelegated } from "./handlers/heartbeat-delegated.js";
|
|
28
28
|
import { handleCancelDelegated } from "./handlers/cancel-delegated.js";
|
|
29
29
|
import { handleDiffusionStream } from "./handlers/diffusion-stream.js";
|
|
30
|
+
import { handleVideoStream } from "./handlers/video-stream.js";
|
|
31
|
+
import { handleUpscaleStream } from "./handlers/upscale-stream.js";
|
|
32
|
+
import { handleClassify } from "./handlers/classify.js";
|
|
30
33
|
import { handlePluginInvoke, handlePluginInvokeStream, } from "./handlers/plugin-invoke.js";
|
|
31
34
|
import { handleModelRegistryList, handleModelRegistrySearch, handleModelRegistryGetModel, } from "./handlers/registry.js";
|
|
32
35
|
import { handleSuspend } from "./handlers/suspend.js";
|
|
@@ -48,16 +51,33 @@ function isModelDelegated(request) {
|
|
|
48
51
|
const entry = getModelEntry(request.modelId);
|
|
49
52
|
return entry?.isDelegated ?? false;
|
|
50
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Should the cancel be forwarded to a delegated provider?
|
|
56
|
+
*
|
|
57
|
+
* After the 0.11.0 wire-schema collapse the cancel envelope has two
|
|
58
|
+
* operations:
|
|
59
|
+
*
|
|
60
|
+
* - `request` — targeted cancel by `requestId`. Always handled
|
|
61
|
+
* locally: the worker-singleton `RequestRegistry` is the source of
|
|
62
|
+
* truth for active requests (delegated handlers register their own
|
|
63
|
+
* requests on it the same way local handlers do), so a `requestId`
|
|
64
|
+
* cancel always lands on the right worker without needing a hop
|
|
65
|
+
* through the provider. Returning `false` here keeps the cancel on
|
|
66
|
+
* the local cancel handler, where it routes through the registry
|
|
67
|
+
* and (for downloads) the `markClearCacheForRequest` helper.
|
|
68
|
+
*
|
|
69
|
+
* - `broad` — abort every in-flight request on a model. Forwarded to
|
|
70
|
+
* the delegated provider iff the targeted model itself is
|
|
71
|
+
* delegated; the provider then runs the same broad-cancel sweep
|
|
72
|
+
* server-side. Local broad cancels for non-delegated models stay
|
|
73
|
+
* on this worker.
|
|
74
|
+
*/
|
|
51
75
|
function isCancelDelegated(request) {
|
|
52
76
|
if (request.type !== "cancel")
|
|
53
77
|
return false;
|
|
54
|
-
if (request.operation
|
|
55
|
-
return
|
|
56
|
-
|
|
57
|
-
if (request.operation === "downloadAsset") {
|
|
58
|
-
return !!request.delegate;
|
|
59
|
-
}
|
|
60
|
-
return false;
|
|
78
|
+
if (request.operation !== "broad")
|
|
79
|
+
return false;
|
|
80
|
+
return isModelDelegated(request);
|
|
61
81
|
}
|
|
62
82
|
export const registry = {
|
|
63
83
|
// Simple Reply handlers
|
|
@@ -104,6 +124,9 @@ export const registry = {
|
|
|
104
124
|
textToSpeechStream: { type: "duplex", handler: handleTextToSpeechStream },
|
|
105
125
|
ocrStream: { type: "stream", handler: handleOCRStream },
|
|
106
126
|
diffusionStream: { type: "stream", handler: handleDiffusionStream },
|
|
127
|
+
videoStream: { type: "stream", handler: handleVideoStream },
|
|
128
|
+
upscaleStream: { type: "stream", handler: handleUpscaleStream },
|
|
129
|
+
classify: { type: "stream", handler: handleClassify },
|
|
107
130
|
pluginInvokeStream: { type: "stream", handler: handlePluginInvokeStream },
|
|
108
131
|
// Handlers with delegation support
|
|
109
132
|
loadModel: {
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Request } from "../../schemas/index";
|
|
2
|
+
import type { HandlerEntry } from "./handler-utils";
|
|
3
|
+
export interface HandlerSelection {
|
|
4
|
+
handler: HandlerEntry["handler"];
|
|
5
|
+
isDelegated: boolean;
|
|
6
|
+
}
|
|
7
|
+
export declare function selectHandler(entry: HandlerEntry, request: Request): HandlerSelection;
|
|
8
|
+
export declare function handlerSupportsProgress(entry: HandlerEntry, request: Request): boolean;
|
|
9
|
+
//# sourceMappingURL=handler-selection.d.ts.map
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export function selectHandler(entry, request) {
|
|
2
|
+
const isDelegated = !!(entry.delegatedHandler && entry.isDelegated?.(request));
|
|
3
|
+
return {
|
|
4
|
+
handler: isDelegated ? entry.delegatedHandler : entry.handler,
|
|
5
|
+
isDelegated,
|
|
6
|
+
};
|
|
7
|
+
}
|
|
8
|
+
export function handlerSupportsProgress(entry, request) {
|
|
9
|
+
return !!("withProgress" in request &&
|
|
10
|
+
request.withProgress &&
|
|
11
|
+
(typeof entry.supportsProgress === "function"
|
|
12
|
+
? entry.supportsProgress(request)
|
|
13
|
+
: entry.supportsProgress));
|
|
14
|
+
}
|
|
@@ -6,6 +6,7 @@ import { setRuntimeContext } from "../bare/registry/runtime-context-registry.js"
|
|
|
6
6
|
import {} from "./profiling/index.js";
|
|
7
7
|
import { isTerminalChunk } from "./rpc-utils.js";
|
|
8
8
|
import { createProgressThrottle } from "./progress-throttle.js";
|
|
9
|
+
import { handlerSupportsProgress, selectHandler, } from "./handler-selection.js";
|
|
9
10
|
function getProfilingMetaFromRequest(request) {
|
|
10
11
|
if (PROFILING_KEY in request) {
|
|
11
12
|
return request[PROFILING_KEY];
|
|
@@ -74,9 +75,7 @@ async function executeProgressHandler(req, request, handler, profiler, isDelegat
|
|
|
74
75
|
const stream = req.createResponseStream();
|
|
75
76
|
profiler.startHandler();
|
|
76
77
|
const writeBatch = (updates) => {
|
|
77
|
-
const payload = updates
|
|
78
|
-
.map((u) => profiler.serialize(u, false))
|
|
79
|
-
.join("\n");
|
|
78
|
+
const payload = updates.map((u) => profiler.serialize(u, false)).join("\n");
|
|
80
79
|
stream.write(payload + "\n", "utf-8");
|
|
81
80
|
};
|
|
82
81
|
const throttle = createProgressThrottle(writeBatch);
|
|
@@ -105,9 +104,7 @@ async function executeProgressHandler(req, request, handler, profiler, isDelegat
|
|
|
105
104
|
}
|
|
106
105
|
}
|
|
107
106
|
export async function executeDuplexHandler(_req, request, entry, inputStream, outputStream, profiler) {
|
|
108
|
-
const handler = entry
|
|
109
|
-
? entry.delegatedHandler
|
|
110
|
-
: entry.handler;
|
|
107
|
+
const { handler } = selectHandler(entry, request);
|
|
111
108
|
profiler.startHandler();
|
|
112
109
|
try {
|
|
113
110
|
for await (const response of handler(request, inputStream)) {
|
|
@@ -127,13 +124,8 @@ export async function executeDuplexHandler(_req, request, entry, inputStream, ou
|
|
|
127
124
|
}
|
|
128
125
|
// Unified handler executor with delegation and progress support
|
|
129
126
|
export async function executeHandler(req, request, entry, profiler) {
|
|
130
|
-
const isDelegated =
|
|
131
|
-
const
|
|
132
|
-
const wantsProgress = "withProgress" in request &&
|
|
133
|
-
request.withProgress &&
|
|
134
|
-
(typeof entry.supportsProgress === "function"
|
|
135
|
-
? entry.supportsProgress(request)
|
|
136
|
-
: entry.supportsProgress);
|
|
127
|
+
const { handler, isDelegated } = selectHandler(entry, request);
|
|
128
|
+
const wantsProgress = handlerSupportsProgress(entry, request);
|
|
137
129
|
if (entry.type === "duplex") {
|
|
138
130
|
throw new PluginHandlerTypeMismatchError(request.type, "reply or stream", "duplex");
|
|
139
131
|
}
|