@qvac/sdk 0.10.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +1455 -29
- package/README.md +2 -0
- package/dist/client/api/cancel.d.ts +24 -32
- package/dist/client/api/cancel.js +58 -31
- package/dist/client/api/classify.d.ts +23 -0
- package/dist/client/api/classify.js +43 -0
- package/dist/client/api/client-request-id.d.ts +17 -0
- package/dist/client/api/client-request-id.js +27 -0
- package/dist/client/api/completion-stream.d.ts +1 -1
- package/dist/client/api/completion-stream.js +34 -3
- package/dist/client/api/diffusion.d.ts +46 -0
- package/dist/client/api/diffusion.js +50 -1
- package/dist/client/api/download-asset.d.ts +12 -2
- package/dist/client/api/download-asset.js +21 -3
- package/dist/client/api/embed.d.ts +8 -4
- package/dist/client/api/embed.js +14 -1
- package/dist/client/api/get-model-info.d.ts +1 -1
- package/dist/client/api/index.d.ts +5 -0
- package/dist/client/api/index.js +5 -0
- package/dist/client/api/load-model.d.ts +12 -4
- package/dist/client/api/load-model.js +49 -2
- package/dist/client/api/rag.d.ts +9 -3
- package/dist/client/api/rag.js +23 -3
- package/dist/client/api/transcribe.d.ts +23 -8
- package/dist/client/api/transcribe.js +69 -4
- package/dist/client/api/unload-model.d.ts +5 -2
- package/dist/client/api/unload-model.js +9 -4
- package/dist/client/api/upscale.d.ts +45 -0
- package/dist/client/api/upscale.js +99 -0
- package/dist/client/api/video.d.ts +64 -0
- package/dist/client/api/video.js +147 -0
- package/dist/client/api/vla-helpers.d.ts +45 -0
- package/dist/client/api/vla-helpers.js +130 -0
- package/dist/client/api/vla.d.ts +65 -0
- package/dist/client/api/vla.js +109 -0
- package/dist/client/config-loader/resolve-config.node.d.ts +16 -0
- package/dist/client/config-loader/resolve-config.node.js +77 -23
- package/dist/client/plugins-factory.d.ts +8 -0
- package/dist/client/plugins-factory.js +10 -0
- package/dist/client/rpc/bare-client.js +62 -66
- package/dist/client/rpc/ensure-worker-ready.d.ts +9 -0
- package/dist/client/rpc/ensure-worker-ready.js +17 -0
- package/dist/client/rpc/expo-rpc-client.js +2 -1
- package/dist/client/rpc/node-rpc-client.js +22 -8
- package/dist/client/rpc/rpc-client.d.ts +2 -2
- package/dist/client/rpc/rpc-client.js +10 -4
- package/dist/client/rpc/rpc-error.d.ts +9 -0
- package/dist/client/rpc/rpc-error.js +105 -0
- package/dist/commands/bundle/bare-pack.d.ts +13 -0
- package/dist/commands/bundle/bare-pack.js +54 -0
- package/dist/commands/bundle/constants.d.ts +7 -0
- package/dist/commands/bundle/constants.js +23 -0
- package/dist/commands/bundle/entry-gen.d.ts +2 -0
- package/dist/commands/bundle/entry-gen.js +48 -0
- package/dist/commands/bundle/index.d.ts +20 -0
- package/dist/commands/bundle/index.js +141 -0
- package/dist/commands/bundle/manifest.d.ts +21 -0
- package/dist/commands/bundle/manifest.js +193 -0
- package/dist/commands/bundle/plugins.d.ts +9 -0
- package/dist/commands/bundle/plugins.js +51 -0
- package/dist/commands/index.d.ts +5 -0
- package/dist/commands/index.js +2 -0
- package/dist/commands/verify/abi.d.ts +52 -0
- package/dist/commands/verify/abi.js +125 -0
- package/dist/commands/verify/addon-source.d.ts +43 -0
- package/dist/commands/verify/addon-source.js +75 -0
- package/dist/commands/verify/bundle-source.d.ts +12 -0
- package/dist/commands/verify/bundle-source.js +53 -0
- package/dist/commands/verify/index.d.ts +59 -0
- package/dist/commands/verify/index.js +371 -0
- package/dist/commands/verify/node-modules-source.d.ts +11 -0
- package/dist/commands/verify/node-modules-source.js +76 -0
- package/dist/commands/verify/prebuilds.d.ts +15 -0
- package/dist/commands/verify/prebuilds.js +39 -0
- package/dist/examples/cancel-by-request-id.d.ts +38 -0
- package/dist/examples/cancel-by-request-id.js +103 -0
- package/dist/examples/classification/classify-image.d.ts +2 -0
- package/dist/examples/classification/classify-image.js +23 -0
- package/dist/examples/diffusion-esrgan-upscale.d.ts +2 -0
- package/dist/examples/diffusion-esrgan-upscale.js +78 -0
- package/dist/examples/diffusion-txt2vid.d.ts +2 -0
- package/dist/examples/diffusion-txt2vid.js +81 -0
- package/dist/examples/download-with-cancel.js +10 -10
- package/dist/examples/embed-multi-gpu.d.ts +2 -0
- package/dist/examples/embed-multi-gpu.js +46 -0
- package/dist/examples/llamacpp-multi-gpu.d.ts +2 -0
- package/dist/examples/llamacpp-multi-gpu.js +57 -0
- package/dist/examples/plugins.js +3 -7
- package/dist/examples/rag/rag-hyperdb/cancellation.js +20 -17
- package/dist/examples/tools/llamacpp-tools-gemma4.d.ts +2 -0
- package/dist/examples/tools/llamacpp-tools-gemma4.js +73 -0
- package/dist/examples/tools/llamacpp-tools-qwen35.d.ts +2 -0
- package/dist/examples/tools/llamacpp-tools-qwen35.js +69 -0
- package/dist/examples/transcription/parakeet-ctc-filesystem.js +18 -12
- package/dist/examples/transcription/parakeet-microphone-record.js +7 -12
- package/dist/examples/transcription/parakeet-microphone-stream.d.ts +2 -0
- package/dist/examples/transcription/parakeet-microphone-stream.js +79 -0
- package/dist/examples/transcription/parakeet-sortformer-streaming.d.ts +2 -0
- package/dist/examples/transcription/parakeet-sortformer-streaming.js +121 -0
- package/dist/examples/transcription/parakeet-sortformer.js +58 -55
- package/dist/examples/transcription/parakeet-tdt-filesystem.js +18 -15
- package/dist/examples/transcription/whispercpp-microphone-conversation.d.ts +2 -0
- package/dist/examples/transcription/whispercpp-microphone-conversation.js +107 -0
- package/dist/examples/tts/chatterbox.js +7 -16
- package/dist/examples/tts/llm-to-tts-streaming.js +3 -10
- package/dist/examples/tts/supertonic-multilingual.js +4 -12
- package/dist/examples/tts/supertonic.js +5 -12
- package/dist/examples/vla-smolvla.d.ts +2 -0
- package/dist/examples/vla-smolvla.js +75 -0
- package/dist/examples/voice-assistant/voice-assistant.js +3 -10
- package/dist/expo/plugins/find-in-ancestor-node-modules.d.ts +14 -0
- package/dist/expo/plugins/find-in-ancestor-node-modules.js +24 -0
- package/dist/expo/plugins/get-project-root.d.ts +22 -0
- package/dist/expo/plugins/get-project-root.js +23 -0
- package/dist/expo/plugins/index.d.ts +4 -0
- package/dist/expo/plugins/index.js +4 -0
- package/dist/expo/plugins/resolve-sdk-package-dir.d.ts +9 -2
- package/dist/expo/plugins/resolve-sdk-package-dir.js +38 -11
- package/dist/expo/plugins/withDeviceInfo.js +2 -1
- package/dist/expo/plugins/withMobileBundle.d.ts +5 -4
- package/dist/expo/plugins/withMobileBundle.js +42 -58
- package/dist/index.d.ts +6 -2
- package/dist/index.js +23 -2
- package/dist/models/registry/models.d.ts +3725 -514
- package/dist/models/registry/models.js +7269 -3910
- package/dist/models/update-models/naming.js +22 -8
- package/dist/pear/pre.d.ts +3 -0
- package/dist/pear/pre.js +9 -2
- package/dist/schemas/cancel.d.ts +172 -28
- package/dist/schemas/cancel.js +120 -19
- package/dist/schemas/classification.d.ts +53 -0
- package/dist/schemas/classification.js +33 -0
- package/dist/schemas/common.d.ts +860 -1379
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +17 -0
- package/dist/schemas/completion-event.js +12 -1
- package/dist/schemas/completion-stream.d.ts +16 -1
- package/dist/schemas/completion-stream.js +25 -7
- package/dist/schemas/download-asset.d.ts +25 -9
- package/dist/schemas/download-asset.js +17 -6
- package/dist/schemas/embed.d.ts +1 -0
- package/dist/schemas/embed.js +5 -0
- package/dist/schemas/engine-addon-map.d.ts +3 -0
- package/dist/schemas/engine-addon-map.js +18 -5
- package/dist/schemas/error.d.ts +31 -0
- package/dist/schemas/error.js +29 -1
- package/dist/schemas/finetune.d.ts +2 -0
- package/dist/schemas/finetune.js +5 -0
- package/dist/schemas/get-model-info.d.ts +10 -6
- package/dist/schemas/get-model-info.js +2 -0
- package/dist/schemas/index.d.ts +3 -0
- package/dist/schemas/index.js +3 -0
- package/dist/schemas/llamacpp-config.d.ts +64 -8
- package/dist/schemas/llamacpp-config.js +12 -0
- package/dist/schemas/load-model.d.ts +4644 -8605
- package/dist/schemas/load-model.js +90 -7
- package/dist/schemas/model-src-utils.d.ts +34 -14
- package/dist/schemas/model-types.d.ts +54 -14
- package/dist/schemas/model-types.js +28 -5
- package/dist/schemas/ocr.d.ts +8 -3
- package/dist/schemas/plugin.d.ts +60 -6
- package/dist/schemas/plugin.js +31 -5
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +17 -9
- package/dist/schemas/registry.d.ts +74 -30
- package/dist/schemas/registry.js +5 -0
- package/dist/schemas/reload-config.d.ts +34 -14
- package/dist/schemas/sdcpp-config.d.ts +445 -16
- package/dist/schemas/sdcpp-config.js +387 -26
- package/dist/schemas/sdk-config.d.ts +2431 -2723
- package/dist/schemas/sdk-config.js +16 -1
- package/dist/schemas/sdk-errors-client.d.ts +6 -0
- package/dist/schemas/sdk-errors-client.js +31 -1
- package/dist/schemas/sdk-errors-server.d.ts +8 -1
- package/dist/schemas/sdk-errors-server.js +45 -5
- package/dist/schemas/text-to-speech.d.ts +81 -698
- package/dist/schemas/text-to-speech.js +46 -19
- package/dist/schemas/transcription-config.d.ts +40 -229
- package/dist/schemas/transcription-config.js +76 -15
- package/dist/schemas/transcription.d.ts +108 -2
- package/dist/schemas/transcription.js +81 -0
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +13 -2
- package/dist/schemas/translation-config.d.ts +90 -40
- package/dist/schemas/unload-model.d.ts +2 -1
- package/dist/schemas/unload-model.js +4 -1
- package/dist/schemas/vla.d.ts +87 -0
- package/dist/schemas/vla.js +108 -0
- package/dist/scripts/run-unit-tests.js +22 -3
- package/dist/server/bare/ops/cancel.d.ts +27 -1
- package/dist/server/bare/ops/cancel.js +36 -10
- package/dist/server/bare/ops/embed.d.ts +3 -3
- package/dist/server/bare/ops/embed.js +144 -28
- package/dist/server/bare/ops/kv-cache-utils.d.ts +0 -7
- package/dist/server/bare/ops/kv-cache-utils.js +8 -47
- package/dist/server/bare/ops/transcribe.d.ts +11 -5
- package/dist/server/bare/ops/transcribe.js +230 -50
- package/dist/server/bare/ops/translate.d.ts +1 -1
- package/dist/server/bare/ops/translate.js +197 -88
- package/dist/server/bare/plugins/ggml-classification/ops/classify.d.ts +6 -0
- package/dist/server/bare/plugins/ggml-classification/ops/classify.js +30 -0
- package/dist/server/bare/plugins/ggml-classification/plugin.d.ts +32 -0
- package/dist/server/bare/plugins/ggml-classification/plugin.js +55 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-hparams.d.ts +3 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-hparams.js +12 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-run.d.ts +3 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-run.js +75 -0
- package/dist/server/bare/plugins/ggml-vla/plugin.d.ts +54 -0
- package/dist/server/bare/plugins/ggml-vla/plugin.js +57 -0
- package/dist/server/bare/plugins/index.d.ts +3 -1
- package/dist/server/bare/plugins/index.js +3 -1
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts +12 -3
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +173 -172
- package/dist/server/bare/plugins/llamacpp-completion/ops/context-overflow.d.ts +29 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/context-overflow.js +65 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/finetune.js +125 -21
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.d.ts +154 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.js +448 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.d.ts +21 -73
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.js +18 -101
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +36 -3
- package/dist/server/bare/plugins/llamacpp-completion/plugin.js +183 -85
- package/dist/server/bare/plugins/llamacpp-completion/transform.d.ts +10 -0
- package/dist/server/bare/plugins/llamacpp-completion/transform.js +34 -0
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.d.ts +7 -0
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +10 -1
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +42 -15
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +5 -1
- package/dist/server/bare/plugins/onnx-ocr/plugin.d.ts +8 -3
- package/dist/server/bare/plugins/onnx-ocr/plugin.js +3 -0
- package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +73 -244
- package/dist/server/bare/plugins/parakeet-transcription/plugin.js +68 -96
- package/dist/server/bare/plugins/sdcpp-generation/ops/diffusion.js +8 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/upscale.d.ts +3 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/upscale.js +40 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/video.d.ts +3 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/video.js +199 -0
- package/dist/server/bare/plugins/sdcpp-generation/plugin.d.ts +288 -15
- package/dist/server/bare/plugins/sdcpp-generation/plugin.js +150 -7
- package/dist/server/bare/plugins/{onnx-tts → tts-ggml}/ops/text-to-speech-stream.js +2 -15
- package/dist/server/bare/plugins/{onnx-tts → tts-ggml}/ops/text-to-speech.js +2 -1
- package/dist/server/bare/plugins/tts-ggml/plugin.d.ts +75 -0
- package/dist/server/bare/plugins/tts-ggml/plugin.js +189 -0
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +47 -3
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.js +61 -14
- package/dist/server/bare/rag-hyperdb/index.d.ts +1 -1
- package/dist/server/bare/rag-hyperdb/index.js +1 -1
- package/dist/server/bare/rag-hyperdb/rag-operation-manager.d.ts +26 -4
- package/dist/server/bare/rag-hyperdb/rag-operation-manager.js +65 -28
- package/dist/server/bare/registry/model-config-utils.js +7 -0
- package/dist/server/bare/runtime/disposable-scope.d.ts +36 -0
- package/dist/server/bare/runtime/disposable-scope.js +69 -0
- package/dist/server/bare/runtime/index.d.ts +8 -0
- package/dist/server/bare/runtime/index.js +3 -0
- package/dist/server/bare/runtime/request-context.d.ts +49 -0
- package/dist/server/bare/runtime/request-context.js +1 -0
- package/dist/server/bare/runtime/request-id.d.ts +12 -0
- package/dist/server/bare/runtime/request-id.js +14 -0
- package/dist/server/bare/runtime/request-registry-singleton.d.ts +4 -0
- package/dist/server/bare/runtime/request-registry-singleton.js +27 -0
- package/dist/server/bare/runtime/request-registry.d.ts +136 -0
- package/dist/server/bare/runtime/request-registry.js +297 -0
- package/dist/server/bare/runtime/with-request-context.d.ts +22 -0
- package/dist/server/bare/runtime/with-request-context.js +53 -0
- package/dist/server/bare/runtime-lifecycle.js +9 -9
- package/dist/server/bare/types/addon-responses.d.ts +12 -1
- package/dist/server/env.js +4 -1
- package/dist/server/plugins/index.d.ts +1 -1
- package/dist/server/plugins/index.js +1 -1
- package/dist/server/plugins/registry.d.ts +1 -0
- package/dist/server/plugins/registry.js +5 -0
- package/dist/server/rpc/handler-registry.js +30 -7
- package/dist/server/rpc/handler-selection.d.ts +9 -0
- package/dist/server/rpc/handler-selection.js +14 -0
- package/dist/server/rpc/handler-utils.js +5 -13
- package/dist/server/rpc/handlers/cancel-delegated.js +28 -31
- package/dist/server/rpc/handlers/cancelHandler.d.ts +30 -1
- package/dist/server/rpc/handlers/cancelHandler.js +54 -22
- package/dist/server/rpc/handlers/classify.d.ts +3 -0
- package/dist/server/rpc/handlers/classify.js +4 -0
- package/dist/server/rpc/handlers/delete-cache.d.ts +9 -0
- package/dist/server/rpc/handlers/delete-cache.js +13 -15
- package/dist/server/rpc/handlers/download-asset.js +133 -40
- package/dist/server/rpc/handlers/index.d.ts +4 -0
- package/dist/server/rpc/handlers/index.js +4 -0
- package/dist/server/rpc/handlers/load-model/download-manager.d.ts +60 -2
- package/dist/server/rpc/handlers/load-model/download-manager.js +141 -8
- package/dist/server/rpc/handlers/load-model/handler.js +216 -98
- package/dist/server/rpc/handlers/load-model/http.js +3 -3
- package/dist/server/rpc/handlers/load-model/hyperdrive.js +1 -1
- package/dist/server/rpc/handlers/load-model/registry.js +1 -1
- package/dist/server/rpc/handlers/load-model/resolve-session.d.ts +20 -1
- package/dist/server/rpc/handlers/load-model/resolve-session.js +6 -5
- package/dist/server/rpc/handlers/load-model/resolve.d.ts +3 -2
- package/dist/server/rpc/handlers/load-model/resolve.js +18 -6
- package/dist/server/rpc/handlers/load-model/types.d.ts +24 -0
- package/dist/server/rpc/handlers/rag.js +133 -15
- package/dist/server/rpc/handlers/upscale-stream.d.ts +3 -0
- package/dist/server/rpc/handlers/upscale-stream.js +4 -0
- package/dist/server/rpc/handlers/video-stream.d.ts +3 -0
- package/dist/server/rpc/handlers/video-stream.js +4 -0
- package/dist/server/rpc/profiling/operation-metrics.js +23 -0
- package/dist/server/utils/completion-normalizer.js +8 -0
- package/dist/server/utils/config-transform.js +1 -1
- package/dist/server/utils/tools/dialect.js +4 -0
- package/dist/server/utils/tools/parser.js +16 -1
- package/dist/server/utils/tools/parsers/gemma4native.d.ts +4 -0
- package/dist/server/utils/tools/parsers/gemma4native.js +60 -0
- package/dist/server/utils/tools/parsers/qwen35.d.ts +4 -0
- package/dist/server/utils/tools/parsers/qwen35.js +120 -0
- package/dist/server/worker.js +14 -11
- package/dist/utils/aggregate-events.d.ts +9 -0
- package/dist/utils/aggregate-events.js +18 -3
- package/dist/utils/decorate-promise.d.ts +34 -0
- package/dist/utils/decorate-promise.js +35 -0
- package/dist/utils/errors-client.d.ts +18 -0
- package/dist/utils/errors-client.js +31 -0
- package/dist/utils/errors-server.d.ts +115 -2
- package/dist/utils/errors-server.js +148 -3
- package/package.json +68 -77
- package/dist/server/bare/plugins/onnx-tts/plugin.d.ts +0 -490
- package/dist/server/bare/plugins/onnx-tts/plugin.js +0 -266
- package/dist/server/bare/plugins/onnx-tts/wav-helper.d.ts +0 -14
- package/dist/server/bare/plugins/onnx-tts/wav-helper.js +0 -107
- /package/dist/server/bare/plugins/{onnx-tts → tts-ggml}/ops/text-to-speech-stream.d.ts +0 -0
- /package/dist/server/bare/plugins/{onnx-tts → tts-ggml}/ops/text-to-speech.d.ts +0 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { type VideoClientParams, type VideoStats } from "../../schemas/index";
|
|
2
|
+
export interface VideoProgressTick {
|
|
3
|
+
step: number;
|
|
4
|
+
totalSteps: number;
|
|
5
|
+
elapsedMs: number;
|
|
6
|
+
}
|
|
7
|
+
export interface VideoResult {
|
|
8
|
+
requestId: string;
|
|
9
|
+
progressStream: AsyncGenerator<VideoProgressTick>;
|
|
10
|
+
outputs: Promise<Uint8Array[]>;
|
|
11
|
+
stats: Promise<VideoStats | undefined>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Generates a video using a loaded video diffusion model.
|
|
15
|
+
*
|
|
16
|
+
* @param params - Video request parameters (model, prompt, dimensions, frame count, fps, sampler, seed, etc.).
|
|
17
|
+
* @returns A result object exposing `requestId` (stable identifier for this in-flight generation), `progressStream` (async iterator of `{ step, totalSteps, elapsedMs }`), `outputs` (promise of the generated video buffers, typically a single AVI file), and `stats` (promise of generation statistics).
|
|
18
|
+
*
|
|
19
|
+
* Only `txt2vid` mode is supported today. On React Native, prefer a `modelId` loaded with a `delegate` since the bundled video diffusion models are too large for typical mobile devices.
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```typescript
|
|
23
|
+
* // Basic txt2vid generation
|
|
24
|
+
* const { outputs, stats } = video({
|
|
25
|
+
* modelId,
|
|
26
|
+
* mode: "txt2vid",
|
|
27
|
+
* prompt: "a cat surfing a wave at sunset",
|
|
28
|
+
* width: 480,
|
|
29
|
+
* height: 832,
|
|
30
|
+
* video_frames: 17, // must satisfy (4*k + 1)
|
|
31
|
+
* fps: 16,
|
|
32
|
+
* });
|
|
33
|
+
* const buffers = await outputs;
|
|
34
|
+
* fs.writeFileSync("output.avi", buffers[0]);
|
|
35
|
+
*
|
|
36
|
+
* // With progress tracking
|
|
37
|
+
* const { progressStream, outputs } = video({
|
|
38
|
+
* modelId,
|
|
39
|
+
* mode: "txt2vid",
|
|
40
|
+
* prompt: "a sunset over the ocean",
|
|
41
|
+
* });
|
|
42
|
+
* for await (const { step, totalSteps } of progressStream) {
|
|
43
|
+
* console.log(`${step}/${totalSteps}`);
|
|
44
|
+
* }
|
|
45
|
+
* const buffers = await outputs;
|
|
46
|
+
*
|
|
47
|
+
* // With control frames (e.g. for guided generation)
|
|
48
|
+
* const frameA = fs.readFileSync("frame-a.png");
|
|
49
|
+
* const frameB = fs.readFileSync("frame-b.png");
|
|
50
|
+
* const { outputs } = video({
|
|
51
|
+
* modelId,
|
|
52
|
+
* mode: "txt2vid",
|
|
53
|
+
* prompt: "smooth transition between scenes",
|
|
54
|
+
* control_frames: [frameA, frameB],
|
|
55
|
+
* });
|
|
56
|
+
*
|
|
57
|
+
* // Cancellation via requestId
|
|
58
|
+
* const { requestId, outputs } = video({ modelId, mode: "txt2vid", prompt: "..." });
|
|
59
|
+
* // ...later
|
|
60
|
+
* await cancel(requestId);
|
|
61
|
+
* ```
|
|
62
|
+
*/
|
|
63
|
+
export declare function video(params: VideoClientParams): VideoResult;
|
|
64
|
+
//# sourceMappingURL=video.d.ts.map
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { videoStreamResponseSchema, } from "../../schemas/index.js";
|
|
2
|
+
import { stream as streamRpc } from "../rpc/rpc-client.js";
|
|
3
|
+
import { generateClientRequestId } from "./client-request-id.js";
|
|
4
|
+
import { decodeBase64, encodeBase64 } from "../../utils/encoding.js";
|
|
5
|
+
/**
|
|
6
|
+
* Generates a video using a loaded video diffusion model.
|
|
7
|
+
*
|
|
8
|
+
* @param params - Video request parameters (model, prompt, dimensions, frame count, fps, sampler, seed, etc.).
|
|
9
|
+
* @returns A result object exposing `requestId` (stable identifier for this in-flight generation), `progressStream` (async iterator of `{ step, totalSteps, elapsedMs }`), `outputs` (promise of the generated video buffers, typically a single AVI file), and `stats` (promise of generation statistics).
|
|
10
|
+
*
|
|
11
|
+
* Only `txt2vid` mode is supported today. On React Native, prefer a `modelId` loaded with a `delegate` since the bundled video diffusion models are too large for typical mobile devices.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* // Basic txt2vid generation
|
|
16
|
+
* const { outputs, stats } = video({
|
|
17
|
+
* modelId,
|
|
18
|
+
* mode: "txt2vid",
|
|
19
|
+
* prompt: "a cat surfing a wave at sunset",
|
|
20
|
+
* width: 480,
|
|
21
|
+
* height: 832,
|
|
22
|
+
* video_frames: 17, // must satisfy (4*k + 1)
|
|
23
|
+
* fps: 16,
|
|
24
|
+
* });
|
|
25
|
+
* const buffers = await outputs;
|
|
26
|
+
* fs.writeFileSync("output.avi", buffers[0]);
|
|
27
|
+
*
|
|
28
|
+
* // With progress tracking
|
|
29
|
+
* const { progressStream, outputs } = video({
|
|
30
|
+
* modelId,
|
|
31
|
+
* mode: "txt2vid",
|
|
32
|
+
* prompt: "a sunset over the ocean",
|
|
33
|
+
* });
|
|
34
|
+
* for await (const { step, totalSteps } of progressStream) {
|
|
35
|
+
* console.log(`${step}/${totalSteps}`);
|
|
36
|
+
* }
|
|
37
|
+
* const buffers = await outputs;
|
|
38
|
+
*
|
|
39
|
+
* // With control frames (e.g. for guided generation)
|
|
40
|
+
* const frameA = fs.readFileSync("frame-a.png");
|
|
41
|
+
* const frameB = fs.readFileSync("frame-b.png");
|
|
42
|
+
* const { outputs } = video({
|
|
43
|
+
* modelId,
|
|
44
|
+
* mode: "txt2vid",
|
|
45
|
+
* prompt: "smooth transition between scenes",
|
|
46
|
+
* control_frames: [frameA, frameB],
|
|
47
|
+
* });
|
|
48
|
+
*
|
|
49
|
+
* // Cancellation via requestId
|
|
50
|
+
* const { requestId, outputs } = video({ modelId, mode: "txt2vid", prompt: "..." });
|
|
51
|
+
* // ...later
|
|
52
|
+
* await cancel(requestId);
|
|
53
|
+
* ```
|
|
54
|
+
*/
|
|
55
|
+
export function video(params) {
|
|
56
|
+
const { control_frames, ...rest } = params;
|
|
57
|
+
const requestId = generateClientRequestId();
|
|
58
|
+
const request = {
|
|
59
|
+
...rest,
|
|
60
|
+
...(control_frames !== undefined && {
|
|
61
|
+
control_frames: control_frames.map(encodeBase64),
|
|
62
|
+
}),
|
|
63
|
+
type: "videoStream",
|
|
64
|
+
requestId,
|
|
65
|
+
};
|
|
66
|
+
let statsResolver = () => { };
|
|
67
|
+
let statsRejecter = () => { };
|
|
68
|
+
const statsPromise = new Promise((resolve, reject) => {
|
|
69
|
+
statsResolver = resolve;
|
|
70
|
+
statsRejecter = reject;
|
|
71
|
+
});
|
|
72
|
+
statsPromise.catch(() => { });
|
|
73
|
+
const progressQueue = [];
|
|
74
|
+
const collectedBuffers = [];
|
|
75
|
+
let progressDone = false;
|
|
76
|
+
let progressResolve = null;
|
|
77
|
+
let streamError = null;
|
|
78
|
+
let outputsResolver = () => { };
|
|
79
|
+
let outputsRejecter = () => { };
|
|
80
|
+
const outputsPromise = new Promise((resolve, reject) => {
|
|
81
|
+
outputsResolver = resolve;
|
|
82
|
+
outputsRejecter = reject;
|
|
83
|
+
});
|
|
84
|
+
outputsPromise.catch(() => { });
|
|
85
|
+
async function processResponses() {
|
|
86
|
+
try {
|
|
87
|
+
for await (const response of streamRpc(request)) {
|
|
88
|
+
if (response &&
|
|
89
|
+
typeof response === "object" &&
|
|
90
|
+
"type" in response &&
|
|
91
|
+
response.type === "videoStream") {
|
|
92
|
+
const parsed = videoStreamResponseSchema.parse(response);
|
|
93
|
+
if (parsed.step != null && parsed.totalSteps != null && parsed.elapsedMs != null) {
|
|
94
|
+
progressQueue.push({
|
|
95
|
+
step: parsed.step,
|
|
96
|
+
totalSteps: parsed.totalSteps,
|
|
97
|
+
elapsedMs: parsed.elapsedMs,
|
|
98
|
+
});
|
|
99
|
+
if (progressResolve) {
|
|
100
|
+
progressResolve();
|
|
101
|
+
progressResolve = null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (parsed.data) {
|
|
105
|
+
collectedBuffers.push(decodeBase64(parsed.data));
|
|
106
|
+
}
|
|
107
|
+
if (parsed.done) {
|
|
108
|
+
statsResolver(parsed.stats);
|
|
109
|
+
outputsResolver(collectedBuffers);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
streamError = error instanceof Error ? error : new Error(String(error));
|
|
116
|
+
statsRejecter(streamError);
|
|
117
|
+
outputsRejecter(streamError);
|
|
118
|
+
}
|
|
119
|
+
progressDone = true;
|
|
120
|
+
if (progressResolve) {
|
|
121
|
+
progressResolve();
|
|
122
|
+
progressResolve = null;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
void processResponses();
|
|
126
|
+
const progressStream = (async function* () {
|
|
127
|
+
while (true) {
|
|
128
|
+
if (progressQueue.length > 0) {
|
|
129
|
+
yield progressQueue.shift();
|
|
130
|
+
}
|
|
131
|
+
else if (progressDone) {
|
|
132
|
+
if (streamError)
|
|
133
|
+
throw streamError;
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
await new Promise((resolve) => { progressResolve = resolve; });
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
})();
|
|
141
|
+
return {
|
|
142
|
+
requestId,
|
|
143
|
+
progressStream,
|
|
144
|
+
outputs: outputsPromise,
|
|
145
|
+
stats: statsPromise,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure-JS helpers for preparing VLA inference inputs on the client side.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors `preprocessImage` / `padState` / `DEFAULT_IMAGE_SIZE` from
|
|
5
|
+
* `@qvac/vla-ggml/addon.js`. Kept inlined (not re-exported from the addon)
|
|
6
|
+
* so the SDK's client surface never loads the addon's native binding —
|
|
7
|
+
* consumers running under Node / Bun / Expo without VLA prebuilds can still
|
|
8
|
+
* use the SDK to drive a remote VLA worker.
|
|
9
|
+
*
|
|
10
|
+
* This file is a verbatim port of the addon's JS implementation; keep them
|
|
11
|
+
* in sync so the wire-format tensors stay byte-identical regardless of
|
|
12
|
+
* where preprocessing runs (consumer process vs worker vs addon).
|
|
13
|
+
*/
|
|
14
|
+
/** Default vision tower image size used by SmolVLA-LIBERO. */
|
|
15
|
+
export declare const VLA_DEFAULT_IMAGE_SIZE = 512;
|
|
16
|
+
type PixelsInput = Float32Array | Uint8Array | number[];
|
|
17
|
+
type ImageLayout = "hwc" | "chw";
|
|
18
|
+
export interface VlaPreprocessImageOptions {
|
|
19
|
+
size?: number;
|
|
20
|
+
layout?: ImageLayout;
|
|
21
|
+
/**
|
|
22
|
+
* Skip the [0,255] vs [0,1] auto-detection heuristic when the caller knows
|
|
23
|
+
* the range. `1` = pixels already in [0,1] (no rescale). `1/255` = pixels
|
|
24
|
+
* in [0,255] (rescale to [0,1]). Any other value (including the literal
|
|
25
|
+
* `'auto'`) falls back to the heuristic.
|
|
26
|
+
*/
|
|
27
|
+
scale?: number | "auto";
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Resize + letterbox + normalize a camera frame to `(3, size, size)` Float32
|
|
31
|
+
* in `[-1, 1]`. Drop-in equivalent of `@qvac/vla-ggml`'s `preprocessImage`.
|
|
32
|
+
*
|
|
33
|
+
* Letterbox places the resized content at the **bottom-right** with padding
|
|
34
|
+
* at top/left (`padLeft = size - newW`, `padTop = size - newH`), matching
|
|
35
|
+
* the reference smolvla.cpp behavior.
|
|
36
|
+
*/
|
|
37
|
+
export declare function vlaPreprocessImage(pixels: PixelsInput, width: number, height: number, opts?: VlaPreprocessImageOptions): Float32Array;
|
|
38
|
+
/**
|
|
39
|
+
* Zero-pad a state vector to `targetDim`. Extra entries are zero-initialised;
|
|
40
|
+
* input longer than `targetDim` raises. Mirrors how smolvla.cpp expects the
|
|
41
|
+
* state tensor (`max_state_dim` = 32 by default).
|
|
42
|
+
*/
|
|
43
|
+
export declare function vlaPadState(state: ArrayLike<number>, targetDim?: number): Float32Array;
|
|
44
|
+
export {};
|
|
45
|
+
//# sourceMappingURL=vla-helpers.d.ts.map
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure-JS helpers for preparing VLA inference inputs on the client side.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors `preprocessImage` / `padState` / `DEFAULT_IMAGE_SIZE` from
|
|
5
|
+
* `@qvac/vla-ggml/addon.js`. Kept inlined (not re-exported from the addon)
|
|
6
|
+
* so the SDK's client surface never loads the addon's native binding —
|
|
7
|
+
* consumers running under Node / Bun / Expo without VLA prebuilds can still
|
|
8
|
+
* use the SDK to drive a remote VLA worker.
|
|
9
|
+
*
|
|
10
|
+
* This file is a verbatim port of the addon's JS implementation; keep them
|
|
11
|
+
* in sync so the wire-format tensors stay byte-identical regardless of
|
|
12
|
+
* where preprocessing runs (consumer process vs worker vs addon).
|
|
13
|
+
*/
|
|
14
|
+
/** Default vision tower image size used by SmolVLA-LIBERO. */
|
|
15
|
+
export const VLA_DEFAULT_IMAGE_SIZE = 512;
|
|
16
|
+
function detectScale(pixels) {
|
|
17
|
+
if (pixels instanceof Uint8Array)
|
|
18
|
+
return 1 / 255;
|
|
19
|
+
// Float/Number arrays: scan a small window to decide whether it's [0,255] or [0,1].
|
|
20
|
+
const limit = Math.min(pixels.length, 256);
|
|
21
|
+
let maxVal = 0;
|
|
22
|
+
for (let i = 0; i < limit; i++) {
|
|
23
|
+
const v = pixels[i];
|
|
24
|
+
if (v > maxVal)
|
|
25
|
+
maxVal = v;
|
|
26
|
+
}
|
|
27
|
+
return maxVal > 1.001 ? 1 / 255 : 1;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Resize + letterbox + normalize a camera frame to `(3, size, size)` Float32
|
|
31
|
+
* in `[-1, 1]`. Drop-in equivalent of `@qvac/vla-ggml`'s `preprocessImage`.
|
|
32
|
+
*
|
|
33
|
+
* Letterbox places the resized content at the **bottom-right** with padding
|
|
34
|
+
* at top/left (`padLeft = size - newW`, `padTop = size - newH`), matching
|
|
35
|
+
* the reference smolvla.cpp behavior.
|
|
36
|
+
*/
|
|
37
|
+
export function vlaPreprocessImage(pixels, width, height, opts = {}) {
|
|
38
|
+
const size = opts.size ?? VLA_DEFAULT_IMAGE_SIZE;
|
|
39
|
+
const layout = opts.layout ?? "hwc";
|
|
40
|
+
if (!Number.isInteger(width) ||
|
|
41
|
+
!Number.isInteger(height) ||
|
|
42
|
+
width <= 0 ||
|
|
43
|
+
height <= 0) {
|
|
44
|
+
throw new TypeError("vlaPreprocessImage: width/height must be positive integers");
|
|
45
|
+
}
|
|
46
|
+
const expected = width * height * 3;
|
|
47
|
+
if (pixels.length !== expected) {
|
|
48
|
+
throw new RangeError(`vlaPreprocessImage: expected ${expected} pixel values, got ${pixels.length}`);
|
|
49
|
+
}
|
|
50
|
+
const normalize = opts.scale === 1 || opts.scale === 1 / 255
|
|
51
|
+
? opts.scale
|
|
52
|
+
: detectScale(pixels);
|
|
53
|
+
// Letterbox target size (aspect-ratio preserving).
|
|
54
|
+
const ratio = Math.max(width / size, height / size);
|
|
55
|
+
const newW = Math.max(1, Math.floor(width / ratio));
|
|
56
|
+
const newH = Math.max(1, Math.floor(height / ratio));
|
|
57
|
+
const padLeft = size - newW;
|
|
58
|
+
const padTop = size - newH;
|
|
59
|
+
const xScale = width / newW;
|
|
60
|
+
const yScale = height / newH;
|
|
61
|
+
// Output starts at -1 so the pad region is already in [-1, 1] and we only
|
|
62
|
+
// need to overwrite the (newH × newW) inner region with the resized content.
|
|
63
|
+
const out = new Float32Array(3 * size * size);
|
|
64
|
+
out.fill(-1);
|
|
65
|
+
const planeStride = size * size;
|
|
66
|
+
const widthHeight = width * height;
|
|
67
|
+
for (let yy = 0; yy < newH; yy++) {
|
|
68
|
+
const yIn = (yy + 0.5) * yScale - 0.5;
|
|
69
|
+
const y0 = Math.max(0, Math.floor(yIn));
|
|
70
|
+
const y1 = Math.min(height - 1, y0 + 1);
|
|
71
|
+
const dy = Math.min(1, Math.max(0, yIn - y0));
|
|
72
|
+
const dyInv = 1 - dy;
|
|
73
|
+
const outY = yy + padTop;
|
|
74
|
+
for (let xx = 0; xx < newW; xx++) {
|
|
75
|
+
const xIn = (xx + 0.5) * xScale - 0.5;
|
|
76
|
+
const x0 = Math.max(0, Math.floor(xIn));
|
|
77
|
+
const x1 = Math.min(width - 1, x0 + 1);
|
|
78
|
+
const dx = Math.min(1, Math.max(0, xIn - x0));
|
|
79
|
+
const dxInv = 1 - dx;
|
|
80
|
+
const outX = xx + padLeft;
|
|
81
|
+
const w00 = dxInv * dyInv;
|
|
82
|
+
const w10 = dx * dyInv;
|
|
83
|
+
const w01 = dxInv * dy;
|
|
84
|
+
const w11 = dx * dy;
|
|
85
|
+
const outIdx = outY * size + outX;
|
|
86
|
+
if (layout === "hwc") {
|
|
87
|
+
const i00 = (y0 * width + x0) * 3;
|
|
88
|
+
const i10 = (y0 * width + x1) * 3;
|
|
89
|
+
const i01 = (y1 * width + x0) * 3;
|
|
90
|
+
const i11 = (y1 * width + x1) * 3;
|
|
91
|
+
for (let c = 0; c < 3; c++) {
|
|
92
|
+
const v = pixels[i00 + c] * w00 +
|
|
93
|
+
pixels[i10 + c] * w10 +
|
|
94
|
+
pixels[i01 + c] * w01 +
|
|
95
|
+
pixels[i11 + c] * w11;
|
|
96
|
+
// Apply scale and shift into [-1, 1] in one fused multiply-add.
|
|
97
|
+
out[c * planeStride + outIdx] = v * normalize * 2 - 1;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
for (let c = 0; c < 3; c++) {
|
|
102
|
+
const plane = c * widthHeight;
|
|
103
|
+
const v = pixels[plane + y0 * width + x0] * w00 +
|
|
104
|
+
pixels[plane + y0 * width + x1] * w10 +
|
|
105
|
+
pixels[plane + y1 * width + x0] * w01 +
|
|
106
|
+
pixels[plane + y1 * width + x1] * w11;
|
|
107
|
+
out[c * planeStride + outIdx] = v * normalize * 2 - 1;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Zero-pad a state vector to `targetDim`. Extra entries are zero-initialised;
|
|
116
|
+
* input longer than `targetDim` raises. Mirrors how smolvla.cpp expects the
|
|
117
|
+
* state tensor (`max_state_dim` = 32 by default).
|
|
118
|
+
*/
|
|
119
|
+
export function vlaPadState(state, targetDim = 32) {
|
|
120
|
+
if (!Number.isInteger(targetDim) || targetDim <= 0) {
|
|
121
|
+
throw new TypeError("vlaPadState: targetDim must be a positive integer");
|
|
122
|
+
}
|
|
123
|
+
if (state.length > targetDim) {
|
|
124
|
+
throw new RangeError(`vlaPadState: input length ${state.length} exceeds targetDim ${targetDim}`);
|
|
125
|
+
}
|
|
126
|
+
const out = new Float32Array(targetDim);
|
|
127
|
+
for (let i = 0; i < state.length; i++)
|
|
128
|
+
out[i] = state[i];
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { type VlaClientRunParams, type VlaClientRunResult, type VlaHparams } from "../../schemas/index";
|
|
2
|
+
/**
|
|
3
|
+
* Run SmolVLA inference on a loaded VLA model and return the produced action
|
|
4
|
+
* chunk plus per-stage timings.
|
|
5
|
+
*
|
|
6
|
+
* @param params - Inference inputs.
|
|
7
|
+
* @param params.modelId - Identifier of the loaded VLA model (returned by
|
|
8
|
+
* `loadModel({ modelType: "vla", ... })`).
|
|
9
|
+
* @param params.images - One or more preprocessed camera frames; each is a
|
|
10
|
+
* `Float32Array` of length `3 * imgWidth * imgHeight` in CHW layout, range
|
|
11
|
+
* `[-1, 1]`. Use the addon's `preprocessImage()` (re-exported as
|
|
12
|
+
* `vlaPreprocessImage`) to produce them.
|
|
13
|
+
* @param params.imgWidth - Width of each preprocessed image; must equal
|
|
14
|
+
* `hparams.visionImageSize`.
|
|
15
|
+
* @param params.imgHeight - Height of each preprocessed image; must equal
|
|
16
|
+
* `hparams.visionImageSize`.
|
|
17
|
+
* @param params.state - Robot end-effector / gripper state padded to
|
|
18
|
+
* `hparams.maxStateDim` (use `vlaPadState`).
|
|
19
|
+
* @param params.tokens - Tokenized instruction (`Int32Array` of length
|
|
20
|
+
* `hparams.tokenizerMaxLength`). Tokenize on the consumer side with the
|
|
21
|
+
* SmolVLM2 tokenizer.
|
|
22
|
+
* @param params.mask - Token attention mask (`Uint8Array` matching `tokens`).
|
|
23
|
+
* @param params.noise - Optional seeded noise prior
|
|
24
|
+
* (`Float32Array` of length `hparams.chunkSize * hparams.maxActionDim`).
|
|
25
|
+
* When omitted the addon samples its own prior.
|
|
26
|
+
* @returns A `VlaClientRunResult` with the produced `actions` Float32Array
|
|
27
|
+
* (length `chunkSize * actionDim`), the corresponding `chunkSize` /
|
|
28
|
+
* `actionDim` returned by the addon, and optional per-stage `stats`.
|
|
29
|
+
*
|
|
30
|
+
* @example
|
|
31
|
+
* ```typescript
|
|
32
|
+
* import { loadModel, vla, vlaPreprocessImage, vlaPadState, vlaHparams } from "@qvac/sdk";
|
|
33
|
+
*
|
|
34
|
+
* const modelId = await loadModel({ modelSrc: "/path/to/smolvla.gguf", modelType: "vla" });
|
|
35
|
+
* const { hparams } = await vlaHparams({ modelId });
|
|
36
|
+
* const size = hparams.visionImageSize;
|
|
37
|
+
* const front = vlaPreprocessImage(frontPixels, frontW, frontH, { size });
|
|
38
|
+
* const wrist = vlaPreprocessImage(wristPixels, wristW, wristH, { size });
|
|
39
|
+
* const state = vlaPadState(robotState, hparams.maxStateDim);
|
|
40
|
+
* const tokens = new Int32Array(hparams.tokenizerMaxLength);
|
|
41
|
+
* const mask = new Uint8Array(hparams.tokenizerMaxLength);
|
|
42
|
+
* // ...tokenize the instruction into tokens/mask...
|
|
43
|
+
* const { actions } = await vla({
|
|
44
|
+
* modelId, images: [front, wrist], imgWidth: size, imgHeight: size,
|
|
45
|
+
* state, tokens, mask,
|
|
46
|
+
* });
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
export declare function vla(params: VlaClientRunParams): Promise<VlaClientRunResult>;
|
|
50
|
+
/**
|
|
51
|
+
* Fetch the loaded VLA model's hyperparameters and the active ggml backend
|
|
52
|
+
* name. Useful to size token / state / noise buffers before calling `vla()`.
|
|
53
|
+
*
|
|
54
|
+
* @param params - Identifier of the loaded VLA model.
|
|
55
|
+
* @returns The model's hparams and the human-readable backend name
|
|
56
|
+
* (`"CPU"` / `"Vulkan"` / `"Metal"` / `"OpenCL"` / `null` if the addon
|
|
57
|
+
* has not surfaced one).
|
|
58
|
+
*/
|
|
59
|
+
export declare function vlaHparams(params: {
|
|
60
|
+
modelId: string;
|
|
61
|
+
}): Promise<{
|
|
62
|
+
hparams: VlaHparams;
|
|
63
|
+
backendName: string | null;
|
|
64
|
+
}>;
|
|
65
|
+
//# sourceMappingURL=vla.d.ts.map
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { vlaHparamsResponseSchema, vlaRunResponseSchema, } from "../../schemas/index.js";
|
|
2
|
+
import { decodeBase64, encodeBase64 } from "../../utils/encoding.js";
|
|
3
|
+
import { invokePlugin } from "./invoke-plugin.js";
|
|
4
|
+
const VLA_RUN_HANDLER = "vlaRun";
|
|
5
|
+
const VLA_HPARAMS_HANDLER = "vlaHparams";
|
|
6
|
+
function bytesOf(arr) {
|
|
7
|
+
return new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength);
|
|
8
|
+
}
|
|
9
|
+
function f32FromBase64(b64) {
|
|
10
|
+
const bytes = decodeBase64(b64);
|
|
11
|
+
// base64 round-trip produces a fresh Uint8Array whose buffer starts at 0
|
|
12
|
+
// and isn't shared, so the underlying ArrayBuffer is safe to reinterpret.
|
|
13
|
+
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Run SmolVLA inference on a loaded VLA model and return the produced action
|
|
17
|
+
* chunk plus per-stage timings.
|
|
18
|
+
*
|
|
19
|
+
* @param params - Inference inputs.
|
|
20
|
+
* @param params.modelId - Identifier of the loaded VLA model (returned by
|
|
21
|
+
* `loadModel({ modelType: "vla", ... })`).
|
|
22
|
+
* @param params.images - One or more preprocessed camera frames; each is a
|
|
23
|
+
* `Float32Array` of length `3 * imgWidth * imgHeight` in CHW layout, range
|
|
24
|
+
* `[-1, 1]`. Use the addon's `preprocessImage()` (re-exported as
|
|
25
|
+
* `vlaPreprocessImage`) to produce them.
|
|
26
|
+
* @param params.imgWidth - Width of each preprocessed image; must equal
|
|
27
|
+
* `hparams.visionImageSize`.
|
|
28
|
+
* @param params.imgHeight - Height of each preprocessed image; must equal
|
|
29
|
+
* `hparams.visionImageSize`.
|
|
30
|
+
* @param params.state - Robot end-effector / gripper state padded to
|
|
31
|
+
* `hparams.maxStateDim` (use `vlaPadState`).
|
|
32
|
+
* @param params.tokens - Tokenized instruction (`Int32Array` of length
|
|
33
|
+
* `hparams.tokenizerMaxLength`). Tokenize on the consumer side with the
|
|
34
|
+
* SmolVLM2 tokenizer.
|
|
35
|
+
* @param params.mask - Token attention mask (`Uint8Array` matching `tokens`).
|
|
36
|
+
* @param params.noise - Optional seeded noise prior
|
|
37
|
+
* (`Float32Array` of length `hparams.chunkSize * hparams.maxActionDim`).
|
|
38
|
+
* When omitted the addon samples its own prior.
|
|
39
|
+
* @returns A `VlaClientRunResult` with the produced `actions` Float32Array
|
|
40
|
+
* (length `chunkSize * actionDim`), the corresponding `chunkSize` /
|
|
41
|
+
* `actionDim` returned by the addon, and optional per-stage `stats`.
|
|
42
|
+
*
|
|
43
|
+
* @example
|
|
44
|
+
* ```typescript
|
|
45
|
+
* import { loadModel, vla, vlaPreprocessImage, vlaPadState, vlaHparams } from "@qvac/sdk";
|
|
46
|
+
*
|
|
47
|
+
* const modelId = await loadModel({ modelSrc: "/path/to/smolvla.gguf", modelType: "vla" });
|
|
48
|
+
* const { hparams } = await vlaHparams({ modelId });
|
|
49
|
+
* const size = hparams.visionImageSize;
|
|
50
|
+
* const front = vlaPreprocessImage(frontPixels, frontW, frontH, { size });
|
|
51
|
+
* const wrist = vlaPreprocessImage(wristPixels, wristW, wristH, { size });
|
|
52
|
+
* const state = vlaPadState(robotState, hparams.maxStateDim);
|
|
53
|
+
* const tokens = new Int32Array(hparams.tokenizerMaxLength);
|
|
54
|
+
* const mask = new Uint8Array(hparams.tokenizerMaxLength);
|
|
55
|
+
* // ...tokenize the instruction into tokens/mask...
|
|
56
|
+
* const { actions } = await vla({
|
|
57
|
+
* modelId, images: [front, wrist], imgWidth: size, imgHeight: size,
|
|
58
|
+
* state, tokens, mask,
|
|
59
|
+
* });
|
|
60
|
+
* ```
|
|
61
|
+
*/
|
|
62
|
+
export async function vla(params) {
|
|
63
|
+
const wireRequest = {
|
|
64
|
+
type: "vlaRun",
|
|
65
|
+
modelId: params.modelId,
|
|
66
|
+
images: params.images.map((img) => encodeBase64(bytesOf(img))),
|
|
67
|
+
imgWidth: params.imgWidth,
|
|
68
|
+
imgHeight: params.imgHeight,
|
|
69
|
+
state: encodeBase64(bytesOf(params.state)),
|
|
70
|
+
tokens: encodeBase64(bytesOf(params.tokens)),
|
|
71
|
+
mask: encodeBase64(bytesOf(params.mask)),
|
|
72
|
+
...(params.noise !== undefined && {
|
|
73
|
+
noise: encodeBase64(bytesOf(params.noise)),
|
|
74
|
+
}),
|
|
75
|
+
};
|
|
76
|
+
const result = await invokePlugin({
|
|
77
|
+
modelId: params.modelId,
|
|
78
|
+
handler: VLA_RUN_HANDLER,
|
|
79
|
+
params: wireRequest,
|
|
80
|
+
});
|
|
81
|
+
const parsed = vlaRunResponseSchema.parse(result);
|
|
82
|
+
return {
|
|
83
|
+
actions: f32FromBase64(parsed.actions),
|
|
84
|
+
actionDim: parsed.actionDim,
|
|
85
|
+
chunkSize: parsed.chunkSize,
|
|
86
|
+
...(parsed.stats && { stats: parsed.stats }),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Fetch the loaded VLA model's hyperparameters and the active ggml backend
|
|
91
|
+
* name. Useful to size token / state / noise buffers before calling `vla()`.
|
|
92
|
+
*
|
|
93
|
+
* @param params - Identifier of the loaded VLA model.
|
|
94
|
+
* @returns The model's hparams and the human-readable backend name
|
|
95
|
+
* (`"CPU"` / `"Vulkan"` / `"Metal"` / `"OpenCL"` / `null` if the addon
|
|
96
|
+
* has not surfaced one).
|
|
97
|
+
*/
|
|
98
|
+
export async function vlaHparams(params) {
|
|
99
|
+
const wireRequest = {
|
|
100
|
+
type: "vlaHparams",
|
|
101
|
+
modelId: params.modelId,
|
|
102
|
+
};
|
|
103
|
+
const result = await invokePlugin({
|
|
104
|
+
modelId: params.modelId,
|
|
105
|
+
handler: VLA_HPARAMS_HANDLER,
|
|
106
|
+
params: wireRequest,
|
|
107
|
+
});
|
|
108
|
+
return vlaHparamsResponseSchema.parse(result);
|
|
109
|
+
}
|
|
@@ -1,4 +1,20 @@
|
|
|
1
1
|
import { type QvacConfig } from "./config-utils";
|
|
2
|
+
/** Config filenames searched under a project root (bundle/verify/commands). */
|
|
3
|
+
export declare const CONFIG_CANDIDATES: readonly ["qvac.config.ts", "qvac.config.mjs", "qvac.config.js", "qvac.config.json"];
|
|
4
|
+
/** Load and validate a config file at an explicit path (used by RPC init and commands). */
|
|
5
|
+
export declare function loadConfigFromPath(configPath: string): Promise<QvacConfig>;
|
|
6
|
+
/**
|
|
7
|
+
* Resolve a config file path under `projectRoot` (or an explicit relative/absolute path).
|
|
8
|
+
* Used by `bundleSdk` / `verifyBundle` when the caller supplies `projectRoot`.
|
|
9
|
+
*/
|
|
10
|
+
export declare function resolveConfigFileInProject(projectRoot: string, explicitPath?: string): Promise<string | null>;
|
|
11
|
+
/**
|
|
12
|
+
* Load `qvac.config.*` for a known project directory (commands tooling API).
|
|
13
|
+
*/
|
|
14
|
+
export declare function resolveConfigForProject(projectRoot: string, explicitPath?: string): Promise<{
|
|
15
|
+
configPath: string | null;
|
|
16
|
+
config: QvacConfig;
|
|
17
|
+
}>;
|
|
2
18
|
/**
|
|
3
19
|
* Resolution order for Node.js:
|
|
4
20
|
* 1. QVAC_CONFIG_PATH environment variable
|