@qvac/sdk 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +15 -68
- package/dist/client/api/cancel.d.ts +1 -1
- package/dist/client/api/cancel.js +1 -1
- package/dist/client/api/completion-stream.d.ts +51 -12
- package/dist/client/api/completion-stream.js +129 -56
- package/dist/client/api/delete-cache.d.ts +1 -0
- package/dist/client/api/delete-cache.js +1 -0
- package/dist/client/api/diffusion.d.ts +17 -2
- package/dist/client/api/diffusion.js +21 -4
- package/dist/client/api/embed.d.ts +4 -0
- package/dist/client/api/finetune.d.ts +21 -0
- package/dist/client/api/get-loaded-model-info.d.ts +25 -0
- package/dist/client/api/get-loaded-model-info.js +36 -0
- package/dist/client/api/get-model-info.d.ts +9 -1
- package/dist/client/api/get-model-info.js +8 -0
- package/dist/client/api/heartbeat.d.ts +2 -2
- package/dist/client/api/heartbeat.js +2 -2
- package/dist/client/api/index.d.ts +3 -1
- package/dist/client/api/index.js +3 -1
- package/dist/client/api/invoke-plugin.d.ts +16 -0
- package/dist/client/api/invoke-plugin.js +16 -0
- package/dist/client/api/load-model.d.ts +36 -3
- package/dist/client/api/load-model.js +31 -10
- package/dist/client/api/provide.d.ts +11 -5
- package/dist/client/api/provide.js +11 -6
- package/dist/client/api/rag.d.ts +9 -0
- package/dist/client/api/rag.js +9 -0
- package/dist/client/api/registry.d.ts +26 -0
- package/dist/client/api/registry.js +26 -0
- package/dist/client/api/resume.d.ts +20 -4
- package/dist/client/api/resume.js +20 -4
- package/dist/client/api/state.d.ts +23 -0
- package/dist/client/api/state.js +29 -0
- package/dist/client/api/stop-provider.d.ts +11 -5
- package/dist/client/api/stop-provider.js +11 -5
- package/dist/client/api/suspend.d.ts +21 -3
- package/dist/client/api/suspend.js +21 -3
- package/dist/client/api/text-to-speech.d.ts +64 -6
- package/dist/client/api/text-to-speech.js +423 -47
- package/dist/client/api/transcribe.d.ts +29 -4
- package/dist/client/api/transcribe.js +86 -28
- package/dist/client/api/translate.d.ts +1 -0
- package/dist/client/api/translate.js +1 -0
- package/dist/client/rpc/bare-client.js +22 -1
- package/dist/client/rpc/expo-rpc-client.d.ts +1 -1
- package/dist/client/rpc/expo-rpc-client.js +96 -4
- package/dist/client/rpc/node-rpc-client.js +36 -0
- package/dist/examples/audio/mic-input.d.ts +9 -0
- package/dist/examples/audio/mic-input.js +111 -0
- package/dist/examples/completion-events.d.ts +18 -0
- package/dist/examples/completion-events.js +79 -0
- package/dist/examples/delegated-inference/composite.js +12 -18
- package/dist/examples/delegated-inference/consumer-profiled.js +4 -11
- package/dist/examples/delegated-inference/consumer.js +8 -13
- package/dist/examples/delegated-inference/provider.js +11 -16
- package/dist/examples/diffusion-flux2-klein-img2img.d.ts +2 -0
- package/dist/examples/diffusion-flux2-klein-img2img.js +60 -0
- package/dist/examples/diffusion-flux2-klein.js +1 -0
- package/dist/examples/diffusion-img2img.d.ts +2 -0
- package/dist/examples/diffusion-img2img.js +49 -0
- package/dist/examples/diffusion-txt2img.js +2 -1
- package/dist/examples/kv-cache-example.js +11 -12
- package/dist/examples/llamacpp-cache.d.ts +2 -0
- package/dist/examples/llamacpp-cache.js +107 -0
- package/dist/examples/llamacpp-dynamic-tools.d.ts +2 -0
- package/dist/examples/llamacpp-dynamic-tools.js +144 -0
- package/dist/examples/llamacpp-native-tools.js +2 -8
- package/dist/examples/llamacpp-structured-output.d.ts +2 -0
- package/dist/examples/llamacpp-structured-output.js +106 -0
- package/dist/examples/llamacpp-tools-pythonic.d.ts +2 -0
- package/dist/examples/llamacpp-tools-pythonic.js +69 -0
- package/dist/examples/plugins.js +4 -4
- package/dist/examples/suspend-resume.js +21 -4
- package/dist/examples/transcription/parakeet-microphone-record.js +6 -33
- package/dist/examples/transcription/whispercpp-filesystem-streaming.js +7 -5
- package/dist/examples/transcription/whispercpp-filesystem.js +12 -3
- package/dist/examples/transcription/whispercpp-microphone-record.js +6 -29
- package/dist/examples/translation/translation-stream.js +5 -5
- package/dist/examples/tts/llm-to-tts-streaming.d.ts +16 -0
- package/dist/examples/tts/llm-to-tts-streaming.js +143 -0
- package/dist/examples/tts/supertonic-multilingual.js +3 -3
- package/dist/examples/tts/supertonic.js +3 -3
- package/dist/examples/tts/utils.d.ts +11 -1
- package/dist/examples/tts/utils.js +70 -17
- package/dist/examples/voice-assistant/voice-assistant.d.ts +2 -0
- package/dist/examples/voice-assistant/voice-assistant.js +207 -0
- package/dist/expo/plugins/withMobileBundle.js +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/logging/logger.d.ts +11 -0
- package/dist/logging/logger.js +11 -0
- package/dist/logging/types.d.ts +18 -0
- package/dist/models/registry/models.d.ts +6620 -6185
- package/dist/models/registry/models.js +6296 -5318
- package/dist/models/update-models/codegen.js +84 -23
- package/dist/models/update-models/companions.d.ts +18 -0
- package/dist/models/update-models/companions.js +175 -0
- package/dist/models/update-models/naming.js +0 -25
- package/dist/models/update-models/registry.js +4 -2
- package/dist/models/update-models/types.d.ts +37 -9
- package/dist/profiling/index.d.ts +49 -0
- package/dist/profiling/index.js +49 -0
- package/dist/schemas/cancel.d.ts +6 -2
- package/dist/schemas/cancel.js +14 -7
- package/dist/schemas/common.d.ts +361 -279
- package/dist/schemas/common.js +26 -5
- package/dist/schemas/completion-event.d.ts +225 -0
- package/dist/schemas/completion-event.js +79 -0
- package/dist/schemas/completion-stream.d.ts +122 -31
- package/dist/schemas/completion-stream.js +137 -36
- package/dist/schemas/delegate.d.ts +0 -2
- package/dist/schemas/delegate.js +25 -6
- package/dist/schemas/download-asset.d.ts +5 -5
- package/dist/schemas/embed.js +21 -9
- package/dist/schemas/engine-addon-map.d.ts +11 -1
- package/dist/schemas/engine-addon-map.js +2 -2
- package/dist/schemas/finetune.d.ts +16 -16
- package/dist/schemas/finetune.js +8 -8
- package/dist/schemas/get-loaded-model-info.d.ts +52 -0
- package/dist/schemas/get-loaded-model-info.js +44 -0
- package/dist/schemas/get-model-info.d.ts +2 -2
- package/dist/schemas/get-model-info.js +103 -32
- package/dist/schemas/heartbeat.d.ts +0 -1
- package/dist/schemas/index.d.ts +4 -1
- package/dist/schemas/index.js +4 -1
- package/dist/schemas/llamacpp-config.d.ts +14 -4
- package/dist/schemas/llamacpp-config.js +5 -0
- package/dist/schemas/load-model.d.ts +3233 -1260
- package/dist/schemas/load-model.js +23 -16
- package/dist/schemas/model-src-utils.d.ts +7 -6
- package/dist/schemas/model-src-utils.js +27 -1
- package/dist/schemas/model-types.d.ts +4 -4
- package/dist/schemas/ocr.d.ts +1 -1
- package/dist/schemas/plugin.d.ts +27 -3
- package/dist/schemas/plugin.js +27 -2
- package/dist/schemas/profiling.js +11 -3
- package/dist/schemas/provide.d.ts +1 -3
- package/dist/schemas/provide.js +0 -3
- package/dist/schemas/rag.d.ts +4 -4
- package/dist/schemas/registry.d.ts +10 -10
- package/dist/schemas/registry.js +36 -14
- package/dist/schemas/reload-config.d.ts +73 -7
- package/dist/schemas/sdcpp-config.d.ts +16 -8
- package/dist/schemas/sdcpp-config.js +118 -25
- package/dist/schemas/sdk-config.d.ts +116 -90
- package/dist/schemas/sdk-config.js +16 -0
- package/dist/schemas/sdk-errors-client.d.ts +3 -0
- package/dist/schemas/sdk-errors-client.js +16 -1
- package/dist/schemas/sdk-errors-server.d.ts +3 -0
- package/dist/schemas/sdk-errors-server.js +23 -0
- package/dist/schemas/state.d.ts +23 -0
- package/dist/schemas/state.js +14 -0
- package/dist/schemas/stop-provide.d.ts +0 -5
- package/dist/schemas/stop-provide.js +2 -6
- package/dist/schemas/text-to-speech.d.ts +93 -27
- package/dist/schemas/text-to-speech.js +27 -0
- package/dist/schemas/tools.d.ts +16 -0
- package/dist/schemas/tools.js +15 -0
- package/dist/schemas/transcription-config.d.ts +8 -8
- package/dist/schemas/transcription.d.ts +33 -0
- package/dist/schemas/transcription.js +9 -0
- package/dist/schemas/translate.js +59 -19
- package/dist/schemas/translation-config.d.ts +157 -133
- package/dist/schemas/translation-config.js +34 -11
- package/dist/server/bare/delegate-rpc-client.d.ts +1 -1
- package/dist/server/bare/delegate-rpc-client.js +131 -122
- package/dist/server/bare/hyperswarm.d.ts +2 -2
- package/dist/server/bare/hyperswarm.js +12 -7
- package/dist/server/bare/ops/cancel.js +7 -0
- package/dist/server/bare/ops/kv-cache-utils.d.ts +7 -10
- package/dist/server/bare/ops/kv-cache-utils.js +31 -40
- package/dist/server/bare/ops/load-model.js +0 -1
- package/dist/server/bare/ops/transcribe.d.ts +11 -4
- package/dist/server/bare/ops/transcribe.js +25 -4
- package/dist/server/bare/ops/translate.js +42 -4
- package/dist/server/bare/ops/unload-model.js +21 -19
- package/dist/server/bare/plugins/llamacpp-completion/ops/cache-logger.js +1 -2
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts +13 -8
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +257 -85
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.d.ts +101 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.js +135 -0
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +83 -26
- package/dist/server/bare/plugins/llamacpp-completion/plugin.js +57 -32
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +10 -16
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +69 -44
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +113 -94
- package/dist/server/bare/plugins/onnx-ocr/plugin.d.ts +1 -1
- package/dist/server/bare/plugins/onnx-ocr/plugin.js +1 -2
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech-stream.js +116 -0
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech.d.ts +2 -3
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech.js +71 -9
- package/dist/server/bare/plugins/onnx-tts/plugin.d.ts +41 -13
- package/dist/server/bare/plugins/onnx-tts/plugin.js +48 -3
- package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +15 -7
- package/dist/server/bare/plugins/parakeet-transcription/plugin.js +5 -6
- package/dist/server/bare/plugins/sdcpp-generation/ops/diffusion.js +6 -0
- package/dist/server/bare/plugins/sdcpp-generation/plugin.d.ts +8 -5
- package/dist/server/bare/plugins/sdcpp-generation/plugin.js +13 -13
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +17 -1
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.js +40 -17
- package/dist/server/bare/registry/config-registry.js +14 -3
- package/dist/server/bare/registry/model-registry.d.ts +8 -10
- package/dist/server/bare/registry/model-registry.js +17 -12
- package/dist/server/bare/registry/registry-client.js +79 -19
- package/dist/server/bare/runtime-lifecycle.d.ts +2 -1
- package/dist/server/bare/runtime-lifecycle.js +13 -2
- package/dist/server/bare/types/addon-responses.d.ts +8 -3
- package/dist/server/bare/utils/transcribe-metadata.d.ts +21 -0
- package/dist/server/bare/utils/transcribe-metadata.js +28 -0
- package/dist/server/bare/utils/tts-stats.d.ts +22 -0
- package/dist/server/bare/utils/tts-stats.js +10 -0
- package/dist/server/rpc/create-server.d.ts +4 -1
- package/dist/server/rpc/create-server.js +5 -1
- package/dist/server/rpc/delegate-transport.js +36 -29
- package/dist/server/rpc/handle-request.js +28 -3
- package/dist/server/rpc/handler-registry.js +7 -1
- package/dist/server/rpc/handler-utils.d.ts +5 -0
- package/dist/server/rpc/handler-utils.js +22 -0
- package/dist/server/rpc/handlers/cancel-delegated.js +3 -5
- package/dist/server/rpc/handlers/cancelHandler.js +4 -5
- package/dist/server/rpc/handlers/completion-stream-delegated.js +14 -5
- package/dist/server/rpc/handlers/delete-cache.js +8 -3
- package/dist/server/rpc/handlers/download-asset.js +3 -24
- package/dist/server/rpc/handlers/get-loaded-model-info.d.ts +3 -0
- package/dist/server/rpc/handlers/get-loaded-model-info.js +39 -0
- package/dist/server/rpc/handlers/get-model-info.js +93 -84
- package/dist/server/rpc/handlers/heartbeat-delegated.js +2 -2
- package/dist/server/rpc/handlers/index.d.ts +6 -0
- package/dist/server/rpc/handlers/index.js +6 -0
- package/dist/server/rpc/handlers/load-model/download-manager.d.ts +35 -9
- package/dist/server/rpc/handlers/load-model/download-manager.js +142 -38
- package/dist/server/rpc/handlers/load-model/download-stats.d.ts +5 -4
- package/dist/server/rpc/handlers/load-model/download-stats.js +49 -8
- package/dist/server/rpc/handlers/load-model/handler.js +31 -50
- package/dist/server/rpc/handlers/load-model/http.d.ts +2 -2
- package/dist/server/rpc/handlers/load-model/http.js +97 -142
- package/dist/server/rpc/handlers/load-model/hyperdrive.d.ts +2 -2
- package/dist/server/rpc/handlers/load-model/hyperdrive.js +55 -74
- package/dist/server/rpc/handlers/load-model/index.d.ts +0 -1
- package/dist/server/rpc/handlers/load-model/index.js +0 -1
- package/dist/server/rpc/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/server/rpc/handlers/load-model/registry-client-options.js +19 -0
- package/dist/server/rpc/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/server/rpc/handlers/load-model/registry-companion-set.js +208 -0
- package/dist/server/rpc/handlers/load-model/registry-download-utils.d.ts +24 -0
- package/dist/server/rpc/handlers/load-model/registry-download-utils.js +114 -0
- package/dist/server/rpc/handlers/load-model/registry.d.ts +2 -20
- package/dist/server/rpc/handlers/load-model/registry.js +109 -381
- package/dist/server/rpc/handlers/load-model/resolve-session.d.ts +15 -0
- package/dist/server/rpc/handlers/load-model/resolve-session.js +61 -0
- package/dist/server/rpc/handlers/load-model/resolve.d.ts +3 -3
- package/dist/server/rpc/handlers/load-model/resolve.js +14 -23
- package/dist/server/rpc/handlers/load-model/types.d.ts +11 -4
- package/dist/server/rpc/handlers/load-model/types.js +28 -1
- package/dist/server/rpc/handlers/load-model-delegated.js +2 -4
- package/dist/server/rpc/handlers/plugin-dispatch.js +9 -5
- package/dist/server/rpc/handlers/plugin-invoke.js +1 -1
- package/dist/server/rpc/handlers/provideHandler/connection.js +13 -2
- package/dist/server/rpc/handlers/provideHandler/handler.js +42 -29
- package/dist/server/rpc/handlers/state.d.ts +3 -0
- package/dist/server/rpc/handlers/state.js +7 -0
- package/dist/server/rpc/handlers/stopProvideHandler.d.ts +2 -2
- package/dist/server/rpc/handlers/stopProvideHandler.js +12 -7
- package/dist/server/rpc/handlers/text-to-speech-stream.d.ts +3 -0
- package/dist/server/rpc/handlers/text-to-speech-stream.js +4 -0
- package/dist/server/rpc/handlers/unload-model-delegated.js +3 -3
- package/dist/server/rpc/profiling/operation-metrics.js +20 -0
- package/dist/server/rpc/rpc-utils.js +1 -1
- package/dist/server/rpc/transport-selector.d.ts +3 -0
- package/dist/server/rpc/transport-selector.js +18 -0
- package/dist/server/utils/cache/auto-history.d.ts +4 -0
- package/dist/server/utils/cache/auto-history.js +22 -0
- package/dist/server/utils/{cache.d.ts → cache/paths.d.ts} +22 -5
- package/dist/server/utils/{cache.js → cache/paths.js} +36 -20
- package/dist/server/utils/cache/types.d.ts +8 -0
- package/dist/server/utils/cache/types.js +1 -0
- package/dist/server/utils/checksum.d.ts +2 -2
- package/dist/server/utils/checksum.js +1 -1
- package/dist/server/utils/clear-storage.d.ts +5 -0
- package/dist/server/utils/clear-storage.js +17 -0
- package/dist/server/utils/completion-normalizer.d.ts +17 -0
- package/dist/server/utils/completion-normalizer.js +301 -0
- package/dist/server/utils/expand-gguf-shards.d.ts +3 -0
- package/dist/server/utils/expand-gguf-shards.js +22 -0
- package/dist/server/utils/formatting.js +1 -1
- package/dist/server/utils/index.d.ts +4 -2
- package/dist/server/utils/index.js +4 -2
- package/dist/server/utils/qvac-paths.d.ts +2 -0
- package/dist/server/utils/qvac-paths.js +5 -0
- package/dist/server/utils/response-format.d.ts +3 -0
- package/dist/server/utils/response-format.js +20 -0
- package/dist/server/utils/tool-integration.d.ts +15 -4
- package/dist/server/utils/tool-integration.js +21 -41
- package/dist/server/utils/tools/dialect.d.ts +8 -0
- package/dist/server/utils/tools/dialect.js +12 -0
- package/dist/server/utils/tools/index.d.ts +3 -0
- package/dist/server/utils/tools/index.js +2 -0
- package/dist/server/utils/tools/parser.d.ts +6 -0
- package/dist/server/utils/tools/parser.js +40 -0
- package/dist/server/utils/tools/parsers/hermes.d.ts +4 -0
- package/dist/server/utils/tools/parsers/hermes.js +100 -0
- package/dist/server/utils/tools/parsers/json.d.ts +6 -0
- package/dist/server/utils/tools/parsers/json.js +115 -0
- package/dist/server/utils/tools/parsers/pythonic.d.ts +4 -0
- package/dist/server/utils/tools/parsers/pythonic.js +267 -0
- package/dist/server/utils/tools/shared.d.ts +18 -0
- package/dist/server/utils/tools/shared.js +54 -0
- package/dist/server/utils/worker-lock.d.ts +3 -0
- package/dist/server/utils/worker-lock.js +67 -0
- package/dist/server/worker-core.d.ts +16 -1
- package/dist/server/worker-core.js +114 -14
- package/dist/utils/aggregate-events.d.ts +16 -0
- package/dist/utils/aggregate-events.js +52 -0
- package/dist/utils/cache-normalize.d.ts +10 -0
- package/dist/utils/cache-normalize.js +23 -0
- package/dist/utils/encoding.d.ts +8 -0
- package/dist/utils/encoding.js +13 -0
- package/dist/utils/errors-client.d.ts +12 -0
- package/dist/utils/errors-client.js +20 -0
- package/dist/utils/errors-server.d.ts +14 -0
- package/dist/utils/errors-server.js +31 -0
- package/dist/utils/load-model-validation.d.ts +7 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/expo/plugins/patches/android-link.mjs +1 -1
- package/expo/plugins/patches/ios-link.mjs +1 -1
- package/package.json +15 -19
- package/dist/examples/translation/translation-opus.d.ts +0 -2
- package/dist/examples/translation/translation-opus.js +0 -33
- package/dist/server/bare/utils/loader-adapter.d.ts +0 -7
- package/dist/server/bare/utils/loader-adapter.js +0 -9
- package/dist/server/utils/model-path.d.ts +0 -5
- package/dist/server/utils/model-path.js +0 -7
- package/dist/server/utils/tool-parser.d.ts +0 -7
- package/dist/server/utils/tool-parser.js +0 -248
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type TranscribeSegment } from "../../../schemas/index";
|
|
2
|
+
export interface WhisperAddonSegment {
|
|
3
|
+
text: string;
|
|
4
|
+
start?: number;
|
|
5
|
+
end?: number;
|
|
6
|
+
toAppend?: boolean;
|
|
7
|
+
id?: number;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Normalize a native whisper-addon segment to the SDK-level `TranscribeSegment`
|
|
11
|
+
* shape exposed to callers: seconds → milliseconds, `toAppend` → `append`,
|
|
12
|
+
* and defaults for optional fields.
|
|
13
|
+
*/
|
|
14
|
+
export declare function toTranscribeSegment(chunk: WhisperAddonSegment): TranscribeSegment;
|
|
15
|
+
/**
|
|
16
|
+
* Guard used by transcription ops when the caller opts into `metadata: true`.
|
|
17
|
+
* Only the whisper engine emits per-segment metadata; any other engine
|
|
18
|
+
* requesting metadata is rejected with `TranscriptionFailedError`.
|
|
19
|
+
*/
|
|
20
|
+
export declare function assertMetadataSupported(modelId: string, engineType: string, metadata: boolean | undefined): void;
|
|
21
|
+
//# sourceMappingURL=transcribe-metadata.d.ts.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { ModelType } from "../../../schemas/index.js";
|
|
2
|
+
import { TranscriptionFailedError } from "../../../utils/errors-server.js";
|
|
3
|
+
/**
|
|
4
|
+
* Normalize a native whisper-addon segment to the SDK-level `TranscribeSegment`
|
|
5
|
+
* shape exposed to callers: seconds → milliseconds, `toAppend` → `append`,
|
|
6
|
+
* and defaults for optional fields.
|
|
7
|
+
*/
|
|
8
|
+
export function toTranscribeSegment(chunk) {
|
|
9
|
+
return {
|
|
10
|
+
text: chunk.text,
|
|
11
|
+
startMs: (chunk.start ?? 0) * 1000,
|
|
12
|
+
endMs: (chunk.end ?? 0) * 1000,
|
|
13
|
+
append: chunk.toAppend ?? false,
|
|
14
|
+
id: chunk.id ?? 0,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Guard used by transcription ops when the caller opts into `metadata: true`.
|
|
19
|
+
* Only the whisper engine emits per-segment metadata; any other engine
|
|
20
|
+
* requesting metadata is rejected with `TranscriptionFailedError`.
|
|
21
|
+
*/
|
|
22
|
+
export function assertMetadataSupported(modelId, engineType, metadata) {
|
|
23
|
+
if (!metadata)
|
|
24
|
+
return;
|
|
25
|
+
if (engineType !== ModelType.whispercppTranscription) {
|
|
26
|
+
throw new TranscriptionFailedError(`metadata mode is not supported on model ${modelId} (engine: ${engineType || "unknown"}); only the whisper engine supports metadata`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { TtsStats } from "../../../schemas/index";
|
|
2
|
+
/**
|
|
3
|
+
* Shared types and utilities for TTS operations.
|
|
4
|
+
* Used by both text-to-speech.ts and text-to-speech-stream.ts.
|
|
5
|
+
*/
|
|
6
|
+
export type TtsStreamChunk = {
|
|
7
|
+
outputArray: ArrayLike<number>;
|
|
8
|
+
chunkIndex?: number;
|
|
9
|
+
sentenceChunk?: string;
|
|
10
|
+
};
|
|
11
|
+
export type TtsOpYield = {
|
|
12
|
+
buffer: number[];
|
|
13
|
+
chunkIndex?: number;
|
|
14
|
+
sentenceChunk?: string;
|
|
15
|
+
};
|
|
16
|
+
export declare function collectTtsStats(response: {
|
|
17
|
+
stats?: {
|
|
18
|
+
audioDurationMs?: number;
|
|
19
|
+
totalSamples?: number;
|
|
20
|
+
};
|
|
21
|
+
}): TtsStats;
|
|
22
|
+
//# sourceMappingURL=tts-stats.d.ts.map
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export function collectTtsStats(response) {
|
|
2
|
+
return {
|
|
3
|
+
...(response.stats?.audioDurationMs !== undefined && {
|
|
4
|
+
audioDuration: response.stats.audioDurationMs,
|
|
5
|
+
}),
|
|
6
|
+
...(response.stats?.totalSamples !== undefined && {
|
|
7
|
+
totalSamples: response.stats.totalSamples,
|
|
8
|
+
}),
|
|
9
|
+
};
|
|
10
|
+
}
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import RPC from "bare-rpc";
|
|
2
2
|
export declare function createBareKitRPCServer(): RPC;
|
|
3
|
-
export
|
|
3
|
+
export interface IPCClientOptions {
|
|
4
|
+
onDisconnect?: () => void;
|
|
5
|
+
}
|
|
6
|
+
export declare function createIPCClient(socketPath: string, options?: IPCClientOptions): RPC;
|
|
4
7
|
//# sourceMappingURL=create-server.d.ts.map
|
|
@@ -8,7 +8,7 @@ export function createBareKitRPCServer() {
|
|
|
8
8
|
.BareKit;
|
|
9
9
|
return new RPC(IPC, handleRequest);
|
|
10
10
|
}
|
|
11
|
-
export function createIPCClient(socketPath) {
|
|
11
|
+
export function createIPCClient(socketPath, options) {
|
|
12
12
|
logger.info(`Connecting to IPC socket at ${socketPath}`);
|
|
13
13
|
const socket = connect(socketPath);
|
|
14
14
|
socket.on("connect", () => {
|
|
@@ -17,5 +17,9 @@ export function createIPCClient(socketPath) {
|
|
|
17
17
|
socket.on("error", (err) => {
|
|
18
18
|
logger.error("IPC client connection error:", err);
|
|
19
19
|
});
|
|
20
|
+
socket.on("close", () => {
|
|
21
|
+
logger.warn("IPC socket closed — parent process likely terminated");
|
|
22
|
+
options?.onDisconnect?.();
|
|
23
|
+
});
|
|
20
24
|
return new RPC(socket, handleRequest);
|
|
21
25
|
}
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* This module provides `send` and `stream` functions for server-side
|
|
5
5
|
* delegation to remote peers (via HyperSwarm RPC).
|
|
6
6
|
*/
|
|
7
|
-
import { requestSchema, responseSchema, PROFILING_KEY, DELEGATION_BREAKDOWN_KEY, OPERATION_EVENT_KEY, } from "../../schemas/index.js";
|
|
8
|
-
import { nowMs, extractProfilingMeta, recordFailure, generateId, } from "../../profiling/index.js";
|
|
7
|
+
import { requestSchema, responseSchema, PROFILING_KEY, PROFILING_TRAILER_KEY, DELEGATION_BREAKDOWN_KEY, OPERATION_EVENT_KEY, } from "../../schemas/index.js";
|
|
8
|
+
import { nowMs, extractProfilingMeta, stripProfilingMeta, recordFailure, generateId, } from "../../profiling/index.js";
|
|
9
9
|
import { withTimeout, withTimeoutStream } from "../../utils/withTimeout.js";
|
|
10
10
|
import { getServerLogger } from "../../logging/index.js";
|
|
11
11
|
import { DelegateProviderError } from "../../utils/errors-server.js";
|
|
@@ -48,7 +48,8 @@ async function sendBase(request, rpc, options, profilingMeta) {
|
|
|
48
48
|
const payload = JSON.stringify(finalRequest);
|
|
49
49
|
req.send(payload, "utf-8");
|
|
50
50
|
const response = await withTimeout(req.reply("utf-8"), options?.timeout);
|
|
51
|
-
const
|
|
51
|
+
const rawPayload = JSON.parse(response?.toString() || "{}");
|
|
52
|
+
const resPayload = responseSchema.parse(stripProfilingMeta(rawPayload));
|
|
52
53
|
logger.debug("[delegate-transport] Response:", { type: resPayload.type });
|
|
53
54
|
checkAndThrowError(resPayload);
|
|
54
55
|
return resPayload;
|
|
@@ -92,12 +93,13 @@ async function sendProfiled(request, rpc, options, profilingMeta) {
|
|
|
92
93
|
const parseStart = nowMs();
|
|
93
94
|
const rawPayload = JSON.parse(response?.toString() || "{}");
|
|
94
95
|
timings.responseJsonParseMs = nowMs() - parseStart;
|
|
95
|
-
const
|
|
96
|
+
const serverMeta = extractProfilingMeta(rawPayload);
|
|
97
|
+
const cleanPayload = stripProfilingMeta(rawPayload);
|
|
98
|
+
const resPayload = responseSchema.parse(cleanPayload);
|
|
96
99
|
logger.debug("[delegate-transport] Response (profiled):", {
|
|
97
100
|
type: resPayload.type,
|
|
98
101
|
});
|
|
99
102
|
checkAndThrowError(resPayload);
|
|
100
|
-
const serverMeta = extractProfilingMeta(rawPayload);
|
|
101
103
|
const delegationBreakdown = recordDelegationEvents(timings, serverMeta, connectionMs);
|
|
102
104
|
resPayload[DELEGATION_BREAKDOWN_KEY] = delegationBreakdown;
|
|
103
105
|
if (serverMeta?.operation) {
|
|
@@ -150,11 +152,14 @@ async function* streamBase(request, rpc, options = {}, profilingMeta) {
|
|
|
150
152
|
const lines = buffer.split("\n");
|
|
151
153
|
buffer = lines.pop() || ""; // Keep incomplete line in buffer
|
|
152
154
|
for (const line of lines) {
|
|
153
|
-
if (line.trim())
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
155
|
+
if (!line.trim())
|
|
156
|
+
continue;
|
|
157
|
+
const rawPayload = JSON.parse(line);
|
|
158
|
+
if (rawPayload[PROFILING_TRAILER_KEY] === true)
|
|
159
|
+
continue;
|
|
160
|
+
const response = responseSchema.parse(stripProfilingMeta(rawPayload));
|
|
161
|
+
checkAndThrowError(response);
|
|
162
|
+
yield response;
|
|
158
163
|
}
|
|
159
164
|
}
|
|
160
165
|
}
|
|
@@ -198,26 +203,28 @@ async function* streamProfiled(request, rpc, options = {}, profilingMeta) {
|
|
|
198
203
|
const lines = buffer.split("\n");
|
|
199
204
|
buffer = lines.pop() || "";
|
|
200
205
|
for (const line of lines) {
|
|
201
|
-
if (line.trim())
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
yield response;
|
|
206
|
+
if (!line.trim())
|
|
207
|
+
continue;
|
|
208
|
+
const rawPayload = JSON.parse(line);
|
|
209
|
+
const serverMeta = extractProfilingMeta(rawPayload);
|
|
210
|
+
if (serverMeta) {
|
|
211
|
+
lastServerMeta = serverMeta;
|
|
212
|
+
}
|
|
213
|
+
if (rawPayload[PROFILING_TRAILER_KEY] === true)
|
|
214
|
+
continue;
|
|
215
|
+
const response = responseSchema.parse(stripProfilingMeta(rawPayload));
|
|
216
|
+
checkAndThrowError(response);
|
|
217
|
+
timings.chunkCount++;
|
|
218
|
+
if (timings.firstChunkAt === undefined) {
|
|
219
|
+
timings.firstChunkAt = nowMs();
|
|
220
|
+
}
|
|
221
|
+
timings.lastChunkAt = nowMs();
|
|
222
|
+
if (serverMeta?.operation) {
|
|
223
|
+
response[OPERATION_EVENT_KEY] = serverMeta.operation;
|
|
220
224
|
}
|
|
225
|
+
response[DELEGATION_BREAKDOWN_KEY] =
|
|
226
|
+
buildDelegationStreamBreakdown(timings);
|
|
227
|
+
yield response;
|
|
221
228
|
}
|
|
222
229
|
}
|
|
223
230
|
recordDelegationStreamEvents(timings, lastServerMeta);
|
|
@@ -4,11 +4,15 @@ import { resolveModelConfig } from "../bare/registry/model-config-registry.js";
|
|
|
4
4
|
import { sendErrorResponse, sendStreamErrorResponse, } from "../error-handlers.js";
|
|
5
5
|
import { RPCUnknownRequestTypeError, PluginHandlerTypeMismatchError, } from "../../utils/errors-server.js";
|
|
6
6
|
import { registry } from "./handler-registry.js";
|
|
7
|
-
import { executeHandler, executeDuplexHandler, handleInitConfig, isInitConfigMessage, } from "./handler-utils.js";
|
|
7
|
+
import { executeHandler, executeDuplexHandler, handleInitConfig, isInitConfigMessage, handleShutdown, isShutdownMessage, } from "./handler-utils.js";
|
|
8
8
|
import { createServerProfiler } from "./profiling/index.js";
|
|
9
|
+
import { assertLifecycleAllowed } from "../bare/runtime-lifecycle.js";
|
|
10
|
+
import { shouldUseStreamErrorTransport } from "./transport-selector.js";
|
|
9
11
|
export async function handleRequest(req) {
|
|
10
12
|
let profiler;
|
|
11
13
|
let validationStart = 0;
|
|
14
|
+
let rawRequest;
|
|
15
|
+
let entry;
|
|
12
16
|
try {
|
|
13
17
|
const rawData = req.data?.toString();
|
|
14
18
|
// Duplex stream request: metadata arrives as the first chunk on the
|
|
@@ -27,7 +31,21 @@ export async function handleRequest(req) {
|
|
|
27
31
|
handleInitConfig(req, jsonData);
|
|
28
32
|
return;
|
|
29
33
|
}
|
|
34
|
+
// Handle internal pre-terminate cleanup signal (bypasses schema). Lets
|
|
35
|
+
// the client tear addons down while the JS env is still alive so static
|
|
36
|
+
// js_ref_t state doesn't survive into the next worklet's isolate.
|
|
37
|
+
if (isShutdownMessage(jsonData)) {
|
|
38
|
+
await handleShutdown(req);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
30
41
|
const { data: cleanData, profilingMeta } = extractProfilingMeta(jsonData);
|
|
42
|
+
if (cleanData && typeof cleanData === "object") {
|
|
43
|
+
rawRequest = cleanData;
|
|
44
|
+
const rawType = rawRequest["type"];
|
|
45
|
+
if (typeof rawType === "string") {
|
|
46
|
+
entry = registry[rawType];
|
|
47
|
+
}
|
|
48
|
+
}
|
|
31
49
|
profiler = createServerProfiler(profilingMeta);
|
|
32
50
|
profiler.markRequestParsed(jsonParseMs);
|
|
33
51
|
validationStart = nowMs();
|
|
@@ -36,7 +54,8 @@ export async function handleRequest(req) {
|
|
|
36
54
|
attachProfilingMetaToRequest(request, profilingMeta);
|
|
37
55
|
profiler.markRequestValidated(nowMs() - validationStart);
|
|
38
56
|
validationStart = 0;
|
|
39
|
-
|
|
57
|
+
assertLifecycleAllowed(request);
|
|
58
|
+
entry = registry[request.type];
|
|
40
59
|
if (!entry) {
|
|
41
60
|
throw new RPCUnknownRequestTypeError(request.type);
|
|
42
61
|
}
|
|
@@ -46,7 +65,12 @@ export async function handleRequest(req) {
|
|
|
46
65
|
if (profiler && validationStart > 0) {
|
|
47
66
|
profiler.markRequestValidated(nowMs() - validationStart);
|
|
48
67
|
}
|
|
49
|
-
|
|
68
|
+
if (shouldUseStreamErrorTransport(entry, rawRequest)) {
|
|
69
|
+
sendStreamErrorResponse(req.createResponseStream(), error, profiler);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
sendErrorResponse(req, error, profiler);
|
|
73
|
+
}
|
|
50
74
|
}
|
|
51
75
|
}
|
|
52
76
|
function attachProfilingMetaToRequest(request, profilingMeta) {
|
|
@@ -99,6 +123,7 @@ async function handleDuplexRequest(req) {
|
|
|
99
123
|
const request = requestSchema.parse(processedData);
|
|
100
124
|
attachProfilingMetaToRequest(request, profilingMeta);
|
|
101
125
|
profiler.markRequestValidated(nowMs() - validationStart);
|
|
126
|
+
assertLifecycleAllowed(request);
|
|
102
127
|
const entry = registry[request.type];
|
|
103
128
|
if (!entry) {
|
|
104
129
|
throw new RPCUnknownRequestTypeError(request.type);
|
|
@@ -18,7 +18,9 @@ import { stopProvideHandler } from "./handlers/stopProvideHandler.js";
|
|
|
18
18
|
import { handleRag } from "./handlers/rag.js";
|
|
19
19
|
import { handleDeleteCache } from "./handlers/delete-cache.js";
|
|
20
20
|
import { handleTextToSpeech } from "./handlers/text-to-speech.js";
|
|
21
|
+
import { handleTextToSpeechStream } from "./handlers/text-to-speech-stream.js";
|
|
21
22
|
import { handleGetModelInfo } from "./handlers/get-model-info.js";
|
|
23
|
+
import { handleGetLoadedModelInfo } from "./handlers/get-loaded-model-info.js";
|
|
22
24
|
import { handleOCRStream } from "./handlers/ocr-stream.js";
|
|
23
25
|
import { handleHeartbeat } from "./handlers/heartbeat.js";
|
|
24
26
|
import { handleFinetune } from "./handlers/finetune.js";
|
|
@@ -29,6 +31,7 @@ import { handlePluginInvoke, handlePluginInvokeStream, } from "./handlers/plugin
|
|
|
29
31
|
import { handleModelRegistryList, handleModelRegistrySearch, handleModelRegistryGetModel, } from "./handlers/registry.js";
|
|
30
32
|
import { handleSuspend } from "./handlers/suspend.js";
|
|
31
33
|
import { handleResume } from "./handlers/resume.js";
|
|
34
|
+
import { handleState } from "./handlers/state.js";
|
|
32
35
|
function ragSupportsProgress(request) {
|
|
33
36
|
if (request.type !== "rag")
|
|
34
37
|
return false;
|
|
@@ -48,7 +51,7 @@ function isModelDelegated(request) {
|
|
|
48
51
|
function isCancelDelegated(request) {
|
|
49
52
|
if (request.type !== "cancel")
|
|
50
53
|
return false;
|
|
51
|
-
if (request.operation === "inference") {
|
|
54
|
+
if (request.operation === "inference" || request.operation === "embeddings") {
|
|
52
55
|
return isModelDelegated(request);
|
|
53
56
|
}
|
|
54
57
|
if (request.operation === "downloadAsset") {
|
|
@@ -81,6 +84,7 @@ export const registry = {
|
|
|
81
84
|
stopProvide: { type: "reply", handler: stopProvideHandler },
|
|
82
85
|
deleteCache: { type: "reply", handler: handleDeleteCache },
|
|
83
86
|
getModelInfo: { type: "reply", handler: handleGetModelInfo },
|
|
87
|
+
getLoadedModelInfo: { type: "reply", handler: handleGetLoadedModelInfo },
|
|
84
88
|
pluginInvoke: { type: "reply", handler: handlePluginInvoke },
|
|
85
89
|
modelRegistryList: { type: "reply", handler: handleModelRegistryList },
|
|
86
90
|
modelRegistrySearch: { type: "reply", handler: handleModelRegistrySearch },
|
|
@@ -90,12 +94,14 @@ export const registry = {
|
|
|
90
94
|
},
|
|
91
95
|
suspend: { type: "reply", handler: handleSuspend },
|
|
92
96
|
resume: { type: "reply", handler: handleResume },
|
|
97
|
+
state: { type: "reply", handler: handleState },
|
|
93
98
|
// Simple Stream handlers
|
|
94
99
|
transcribe: { type: "stream", handler: handleTranscribe },
|
|
95
100
|
transcribeStream: { type: "duplex", handler: handleTranscribeStream },
|
|
96
101
|
loggingStream: { type: "stream", handler: handleLoggingStream },
|
|
97
102
|
translate: { type: "stream", handler: handleTranslate },
|
|
98
103
|
textToSpeech: { type: "stream", handler: handleTextToSpeech },
|
|
104
|
+
textToSpeechStream: { type: "duplex", handler: handleTextToSpeechStream },
|
|
99
105
|
ocrStream: { type: "stream", handler: handleOCRStream },
|
|
100
106
|
diffusionStream: { type: "stream", handler: handleDiffusionStream },
|
|
101
107
|
pluginInvokeStream: { type: "stream", handler: handlePluginInvokeStream },
|
|
@@ -21,5 +21,10 @@ type InitConfigMessage = {
|
|
|
21
21
|
};
|
|
22
22
|
export declare function isInitConfigMessage(data: unknown): data is InitConfigMessage;
|
|
23
23
|
export declare function handleInitConfig(req: RPC.IncomingRequest, data: InitConfigMessage): void;
|
|
24
|
+
type ShutdownMessage = {
|
|
25
|
+
type: "__shutdown__";
|
|
26
|
+
};
|
|
27
|
+
export declare function isShutdownMessage(data: unknown): data is ShutdownMessage;
|
|
28
|
+
export declare function handleShutdown(req: RPC.IncomingRequest): Promise<void>;
|
|
24
29
|
export {};
|
|
25
30
|
//# sourceMappingURL=handler-utils.d.ts.map
|
|
@@ -170,3 +170,25 @@ export function handleInitConfig(req, data) {
|
|
|
170
170
|
}), "utf-8");
|
|
171
171
|
}
|
|
172
172
|
}
|
|
173
|
+
export function isShutdownMessage(data) {
|
|
174
|
+
return (typeof data === "object" &&
|
|
175
|
+
data !== null &&
|
|
176
|
+
"type" in data &&
|
|
177
|
+
data.type === "__shutdown__");
|
|
178
|
+
}
|
|
179
|
+
export async function handleShutdown(req) {
|
|
180
|
+
try {
|
|
181
|
+
// Lazy import to avoid the import cycle:
|
|
182
|
+
// handler-utils -> worker-core -> create-server -> handle-request
|
|
183
|
+
// -> handler-utils. By the time this runs, all modules are loaded.
|
|
184
|
+
const { cleanupForTerminate } = await import("../worker-core.js");
|
|
185
|
+
await cleanupForTerminate();
|
|
186
|
+
req.reply(JSON.stringify({ success: true }), "utf-8");
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
req.reply(JSON.stringify({
|
|
190
|
+
success: false,
|
|
191
|
+
error: error instanceof Error ? error.message : String(error),
|
|
192
|
+
}), "utf-8");
|
|
193
|
+
}
|
|
194
|
+
}
|
|
@@ -4,13 +4,12 @@ import { getRPC } from "../../bare/delegate-rpc-client.js";
|
|
|
4
4
|
import { send } from "../delegate-transport.js";
|
|
5
5
|
const logger = getServerLogger();
|
|
6
6
|
function resolveDelegationTarget(request) {
|
|
7
|
-
if (request.operation === "inference") {
|
|
7
|
+
if (request.operation === "inference" || request.operation === "embeddings") {
|
|
8
8
|
const entry = getModelEntry(request.modelId);
|
|
9
|
-
if (!entry?.isDelegated
|
|
9
|
+
if (!entry?.isDelegated) {
|
|
10
10
|
return null;
|
|
11
11
|
}
|
|
12
12
|
const target = {
|
|
13
|
-
topic: entry.delegated.topic,
|
|
14
13
|
providerPublicKey: entry.delegated.providerPublicKey,
|
|
15
14
|
};
|
|
16
15
|
if (entry.delegated.timeout !== undefined) {
|
|
@@ -20,7 +19,6 @@ function resolveDelegationTarget(request) {
|
|
|
20
19
|
}
|
|
21
20
|
if (request.operation === "downloadAsset" && request.delegate) {
|
|
22
21
|
const target = {
|
|
23
|
-
topic: request.delegate.topic,
|
|
24
22
|
providerPublicKey: request.delegate.providerPublicKey,
|
|
25
23
|
};
|
|
26
24
|
if (request.delegate.timeout !== undefined) {
|
|
@@ -45,7 +43,7 @@ export async function handleCancelDelegated(request, options) {
|
|
|
45
43
|
return { type: "cancel", success: true };
|
|
46
44
|
}
|
|
47
45
|
try {
|
|
48
|
-
const rpc = await getRPC(target.
|
|
46
|
+
const rpc = await getRPC(target.providerPublicKey, {
|
|
49
47
|
timeout: target.timeout,
|
|
50
48
|
});
|
|
51
49
|
const delegateOpts = {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { cancel } from "../../bare/ops/cancel.js";
|
|
2
|
-
import {
|
|
2
|
+
import { cancelTransfer } from "./load-model/download-manager.js";
|
|
3
3
|
import { cancelRagOperation, DEFAULT_WORKSPACE, } from "../../bare/rag-hyperdb/index.js";
|
|
4
4
|
import { getServerLogger } from "../../../logging/index.js";
|
|
5
5
|
const logger = getServerLogger();
|
|
@@ -7,13 +7,12 @@ export async function cancelHandler(request) {
|
|
|
7
7
|
try {
|
|
8
8
|
switch (request.operation) {
|
|
9
9
|
case "inference":
|
|
10
|
+
case "embeddings":
|
|
10
11
|
await cancel({ modelId: request.modelId });
|
|
11
12
|
break;
|
|
12
|
-
case "downloadAsset":
|
|
13
|
-
|
|
14
|
-
cancelDownload();
|
|
13
|
+
case "downloadAsset":
|
|
14
|
+
cancelTransfer(request.downloadKey, request.clearCache);
|
|
15
15
|
break;
|
|
16
|
-
}
|
|
17
16
|
case "rag": {
|
|
18
17
|
const cancelled = cancelRagOperation(request.workspace);
|
|
19
18
|
if (!cancelled) {
|
|
@@ -8,14 +8,13 @@ export async function* handleCompletionStreamDelegated(request, options) {
|
|
|
8
8
|
const { profilingMeta } = options ?? {};
|
|
9
9
|
// Get delegation info from model registry
|
|
10
10
|
const entry = getModelEntry(request.modelId);
|
|
11
|
-
if (!entry?.isDelegated
|
|
11
|
+
if (!entry?.isDelegated) {
|
|
12
12
|
throw new ModelIsDelegatedError(request.modelId);
|
|
13
13
|
}
|
|
14
|
-
const {
|
|
14
|
+
const { providerPublicKey, timeout, healthCheckTimeout } = entry.delegated;
|
|
15
15
|
try {
|
|
16
16
|
logger.debug(`📤 Sending delegated completionStream request to provider: ${providerPublicKey}${timeout ? `, timeout: ${timeout}ms` : ""}`);
|
|
17
|
-
|
|
18
|
-
const rpc = await getRPC(topic, providerPublicKey, { timeout, healthCheckTimeout });
|
|
17
|
+
const rpc = await getRPC(providerPublicKey, { timeout, healthCheckTimeout });
|
|
19
18
|
// Build delegate options with profiling metadata
|
|
20
19
|
const delegateOpts = { peerKey: providerPublicKey };
|
|
21
20
|
if (profilingMeta) {
|
|
@@ -33,10 +32,20 @@ export async function* handleCompletionStreamDelegated(request, options) {
|
|
|
33
32
|
}
|
|
34
33
|
catch (error) {
|
|
35
34
|
logger.error("Error in delegated completion stream:", error);
|
|
35
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
36
36
|
yield {
|
|
37
37
|
type: "completionStream",
|
|
38
|
-
token: `Error communicating with provider: ${error instanceof Error ? error.message : String(error)}`,
|
|
39
38
|
done: true,
|
|
39
|
+
events: [
|
|
40
|
+
{
|
|
41
|
+
type: "completionDone",
|
|
42
|
+
seq: 0,
|
|
43
|
+
stopReason: "error",
|
|
44
|
+
error: {
|
|
45
|
+
message: `Error communicating with provider: ${message}`,
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
],
|
|
40
49
|
};
|
|
41
50
|
}
|
|
42
51
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { deleteCache as deleteCacheUtil } from "../../bare/ops/kv-cache-utils.js";
|
|
1
|
+
import { clearCacheRegistry, deleteCache as deleteCacheUtil, } from "../../bare/ops/kv-cache-utils.js";
|
|
2
2
|
import { clearCachedMessageCounts } from "../../bare/plugins/llamacpp-completion/ops/completion-stream.js";
|
|
3
3
|
import { getServerLogger } from "../../../logging/index.js";
|
|
4
4
|
const logger = getServerLogger();
|
|
@@ -7,6 +7,7 @@ export async function handleDeleteCache(request) {
|
|
|
7
7
|
if ("all" in request && request.all) {
|
|
8
8
|
await deleteCacheUtil({ all: true });
|
|
9
9
|
clearCachedMessageCounts();
|
|
10
|
+
clearCacheRegistry();
|
|
10
11
|
}
|
|
11
12
|
else if ("kvCacheKey" in request) {
|
|
12
13
|
const params = {
|
|
@@ -15,8 +16,12 @@ export async function handleDeleteCache(request) {
|
|
|
15
16
|
if (request.modelId !== undefined) {
|
|
16
17
|
params.modelId = request.modelId;
|
|
17
18
|
}
|
|
18
|
-
await deleteCacheUtil(params);
|
|
19
|
-
clearCachedMessageCounts();
|
|
19
|
+
const removedPath = await deleteCacheUtil(params);
|
|
20
|
+
clearCachedMessageCounts(removedPath);
|
|
21
|
+
clearCacheRegistry({
|
|
22
|
+
cacheKey: request.kvCacheKey,
|
|
23
|
+
modelId: request.modelId,
|
|
24
|
+
});
|
|
20
25
|
}
|
|
21
26
|
return {
|
|
22
27
|
type: "deleteCache",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { PROFILING_KEY, OPERATION_EVENT_KEY, } from "../../../schemas/index.js";
|
|
2
2
|
import { resolveModelPath, resolveModelPathWithStats, } from "./load-model/resolve.js";
|
|
3
|
+
import { buildDownloadProfilingFields } from "./load-model/types.js";
|
|
3
4
|
import { nowMs, generateProfileId } from "../../../profiling/clock.js";
|
|
4
5
|
import { getServerLogger } from "../../../logging/index.js";
|
|
5
6
|
const logger = getServerLogger();
|
|
@@ -27,30 +28,8 @@ export async function handleDownloadAsset(request, progressCallback) {
|
|
|
27
28
|
if (profilingEnabled) {
|
|
28
29
|
const totalDownloadTimeMs = nowMs() - totalDownloadStart;
|
|
29
30
|
const profileId = profilingMeta?.id ?? generateProfileId();
|
|
30
|
-
const gauges =
|
|
31
|
-
|
|
32
|
-
};
|
|
33
|
-
if (downloadStats) {
|
|
34
|
-
if (downloadStats.downloadTimeMs !== undefined) {
|
|
35
|
-
gauges["downloadTime"] = downloadStats.downloadTimeMs;
|
|
36
|
-
}
|
|
37
|
-
if (downloadStats.totalBytesDownloaded !== undefined) {
|
|
38
|
-
gauges["totalBytesDownloaded"] = downloadStats.totalBytesDownloaded;
|
|
39
|
-
}
|
|
40
|
-
if (downloadStats.downloadSpeedBps !== undefined) {
|
|
41
|
-
gauges["downloadSpeedBps"] = downloadStats.downloadSpeedBps;
|
|
42
|
-
}
|
|
43
|
-
if (downloadStats.checksumValidationTimeMs !== undefined) {
|
|
44
|
-
gauges["checksumValidationTime"] = downloadStats.checksumValidationTimeMs;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
const tags = {};
|
|
48
|
-
if (sourceType) {
|
|
49
|
-
tags["sourceType"] = sourceType;
|
|
50
|
-
}
|
|
51
|
-
if (downloadStats?.cacheHit !== undefined) {
|
|
52
|
-
tags["cacheHit"] = downloadStats.cacheHit ? "true" : "false";
|
|
53
|
-
}
|
|
31
|
+
const { gauges, tags } = buildDownloadProfilingFields(downloadStats, sourceType);
|
|
32
|
+
gauges["totalDownloadTime"] = totalDownloadTimeMs;
|
|
54
33
|
const operationEvent = {
|
|
55
34
|
op: "downloadAsset",
|
|
56
35
|
kind: "handler",
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { getModelEntry } from "../../bare/registry/model-registry.js";
|
|
2
|
+
import { getPlugin } from "../../plugins/registry.js";
|
|
3
|
+
import { ModelNotFoundError } from "../../../utils/errors-server.js";
|
|
4
|
+
import { getServerLogger } from "../../../logging/index.js";
|
|
5
|
+
const logger = getServerLogger();
|
|
6
|
+
export function handleGetLoadedModelInfo(request) {
|
|
7
|
+
const { modelId } = request;
|
|
8
|
+
const entry = getModelEntry(modelId);
|
|
9
|
+
if (!entry) {
|
|
10
|
+
throw new ModelNotFoundError(modelId);
|
|
11
|
+
}
|
|
12
|
+
if (entry.isDelegated) {
|
|
13
|
+
const info = {
|
|
14
|
+
modelId: entry.id,
|
|
15
|
+
isDelegated: true,
|
|
16
|
+
handlers: [],
|
|
17
|
+
providerInfo: {
|
|
18
|
+
providerPublicKey: entry.delegated.providerPublicKey,
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
return { type: "getLoadedModelInfo", info };
|
|
22
|
+
}
|
|
23
|
+
const plugin = getPlugin(entry.local.modelType);
|
|
24
|
+
if (!plugin) {
|
|
25
|
+
logger.warn(`getLoadedModelInfo: no plugin registered for modelType "${entry.local.modelType}" on loaded model "${modelId}"`);
|
|
26
|
+
}
|
|
27
|
+
const info = {
|
|
28
|
+
modelId: entry.id,
|
|
29
|
+
isDelegated: false,
|
|
30
|
+
modelType: entry.local.modelType,
|
|
31
|
+
handlers: plugin ? Object.keys(plugin.handlers) : [],
|
|
32
|
+
loadedAt: entry.local.loadedAt,
|
|
33
|
+
...(plugin && { displayName: plugin.displayName }),
|
|
34
|
+
...(plugin && { addonPackage: plugin.addonPackage }),
|
|
35
|
+
...(entry.local.name && { name: entry.local.name }),
|
|
36
|
+
...(entry.local.path && { path: entry.local.path }),
|
|
37
|
+
};
|
|
38
|
+
return { type: "getLoadedModelInfo", info };
|
|
39
|
+
}
|