@qvac/inference 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +43 -11
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +44 -0
- package/dist/schemas/assess-model-fit.js +40 -0
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +749 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2503 -139
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +28 -25
package/NOTICE
CHANGED
|
@@ -207,6 +207,18 @@ Third-Party Model Licenses
|
|
|
207
207
|
https://huggingface.co/HuggingFaceVLA/smolvla_libero
|
|
208
208
|
SmolVLM2-500M-Video-Instruct-GGUF
|
|
209
209
|
https://huggingface.co/ggml-org/SmolVLM2-500M-Video-Instruct-GGUF
|
|
210
|
+
TranslatePsy-AfriSLM-0.8B-Q4-GGUF
|
|
211
|
+
https://huggingface.co/qvac/TranslatePsy-AfriSLM-0.8B-Q4-GGUF
|
|
212
|
+
TranslatePsy-AfriSLM-0.8B-Q8-GGUF
|
|
213
|
+
https://huggingface.co/qvac/TranslatePsy-AfriSLM-0.8B-Q8-GGUF
|
|
214
|
+
TranslatePsy-AfriSLM-2B-Q4-GGUF
|
|
215
|
+
https://huggingface.co/qvac/TranslatePsy-AfriSLM-2B-Q4-GGUF
|
|
216
|
+
TranslatePsy-AfriSLM-2B-Q8-GGUF
|
|
217
|
+
https://huggingface.co/qvac/TranslatePsy-AfriSLM-2B-Q8-GGUF
|
|
218
|
+
TranslatePsy-AfriSLM-4B-Q4-GGUF
|
|
219
|
+
https://huggingface.co/qvac/TranslatePsy-AfriSLM-4B-Q4-GGUF
|
|
220
|
+
TranslatePsy-AfriSLM-4B-Q8-GGUF
|
|
221
|
+
https://huggingface.co/qvac/TranslatePsy-AfriSLM-4B-Q8-GGUF
|
|
210
222
|
umt5-xxl-enc-q8_0
|
|
211
223
|
https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged
|
|
212
224
|
unlimited-ocr-gguf
|
|
@@ -608,6 +620,12 @@ Third-Party Model Licenses
|
|
|
608
620
|
https://huggingface.co/nvidia/GR00T-N1.7-LIBERO
|
|
609
621
|
groot-q8_vf16
|
|
610
622
|
https://huggingface.co/nvidia/GR00T-N1.7-3B
|
|
623
|
+
nemotron-3.5-asr-streaming-0.6b.f16
|
|
624
|
+
https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
|
|
625
|
+
nemotron-3.5-asr-streaming-0.6b.q4_0
|
|
626
|
+
https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
|
|
627
|
+
nemotron-3.5-asr-streaming-0.6b.q8_0
|
|
628
|
+
https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
|
|
611
629
|
parakeet-unified-en-0.6b.f16
|
|
612
630
|
https://huggingface.co/nvidia/parakeet-unified-en-0.6b
|
|
613
631
|
parakeet-unified-en-0.6b.q4_0
|
|
@@ -663,11 +681,15 @@ JavaScript Dependencies
|
|
|
663
681
|
|
|
664
682
|
--- apache-2.0 (Apache License 2.0) ---
|
|
665
683
|
|
|
666
|
-
@hyperswarm/secret-stream@6.9.
|
|
684
|
+
@hyperswarm/secret-stream@6.9.2
|
|
667
685
|
https://github.com/holepunchto/hyperswarm-secret-stream
|
|
668
686
|
@qvac/error@0.1.1
|
|
687
|
+
@qvac/fabric@0.16.0
|
|
688
|
+
https://github.com/tetherto/qvac
|
|
669
689
|
@qvac/logging@0.1.1
|
|
670
690
|
https://github.com/tetherto/qvac
|
|
691
|
+
@qvac/model-fit@0.12.1
|
|
692
|
+
https://github.com/tetherto/qvac
|
|
671
693
|
@qvac/rag@0.8.1
|
|
672
694
|
https://github.com/tetherto/qvac
|
|
673
695
|
@qvac/registry-client@0.6.1
|
|
@@ -676,7 +698,7 @@ JavaScript Dependencies
|
|
|
676
698
|
https://github.com/tetherto/qvac
|
|
677
699
|
adaptive-timeout@1.0.1
|
|
678
700
|
https://github.com/holepunchto/adaptive-timeout
|
|
679
|
-
b4a@1.
|
|
701
|
+
b4a@1.9.0
|
|
680
702
|
https://github.com/holepunchto/b4a
|
|
681
703
|
bare-abort@2.0.13
|
|
682
704
|
https://github.com/holepunchto/bare-abort
|
|
@@ -694,13 +716,13 @@ JavaScript Dependencies
|
|
|
694
716
|
https://github.com/holepunchto/bare-cpu-info
|
|
695
717
|
bare-crypto@1.15.3
|
|
696
718
|
https://github.com/holepunchto/bare-crypto
|
|
697
|
-
bare-dns@2.2.
|
|
719
|
+
bare-dns@2.2.1
|
|
698
720
|
https://github.com/holepunchto/bare-dns
|
|
699
721
|
bare-env@3.0.1
|
|
700
722
|
https://github.com/holepunchto/bare-env
|
|
701
723
|
bare-events@2.9.2
|
|
702
724
|
https://github.com/holepunchto/bare-events
|
|
703
|
-
bare-fetch@3.
|
|
725
|
+
bare-fetch@3.4.0
|
|
704
726
|
https://github.com/holepunchto/bare-fetch
|
|
705
727
|
bare-form-data@1.2.2
|
|
706
728
|
https://github.com/holepunchto/bare-form-data
|
|
@@ -712,7 +734,7 @@ JavaScript Dependencies
|
|
|
712
734
|
https://github.com/holepunchto/bare-hrtime
|
|
713
735
|
bare-http-parser@2.1.4
|
|
714
736
|
https://github.com/holepunchto/bare-http-parser
|
|
715
|
-
bare-http1@4.6.
|
|
737
|
+
bare-http1@4.6.2
|
|
716
738
|
https://github.com/holepunchto/bare-http1
|
|
717
739
|
bare-https@3.1.0
|
|
718
740
|
https://github.com/holepunchto/bare-https
|
|
@@ -738,6 +760,10 @@ JavaScript Dependencies
|
|
|
738
760
|
https://github.com/holepunchto/bare-process
|
|
739
761
|
bare-rpc@1.3.8
|
|
740
762
|
https://github.com/holepunchto/bare-rpc
|
|
763
|
+
bare-runtime@1.33.4
|
|
764
|
+
https://github.com/holepunchto/bare-runtime
|
|
765
|
+
bare-runtime-darwin-arm64@1.33.4
|
|
766
|
+
https://github.com/holepunchto/bare-runtime
|
|
741
767
|
bare-semver@1.1.0
|
|
742
768
|
https://github.com/holepunchto/bare-semver
|
|
743
769
|
bare-signals@5.0.0
|
|
@@ -746,6 +772,10 @@ JavaScript Dependencies
|
|
|
746
772
|
https://github.com/holepunchto/bare-stdio
|
|
747
773
|
bare-stream@2.13.4
|
|
748
774
|
https://github.com/holepunchto/bare-stream
|
|
775
|
+
bare-structured-clone@1.6.0
|
|
776
|
+
https://github.com/holepunchto/bare-structured-clone
|
|
777
|
+
bare-subprocess@6.1.0
|
|
778
|
+
https://github.com/holepunchto/bare-subprocess
|
|
749
779
|
bare-tcp@2.6.1
|
|
750
780
|
https://github.com/holepunchto/bare-tcp
|
|
751
781
|
bare-tls@3.1.10
|
|
@@ -782,7 +812,7 @@ JavaScript Dependencies
|
|
|
782
812
|
https://github.com/holepunchto/hypercore-id-encoding
|
|
783
813
|
hypercore-stats@2.5.1
|
|
784
814
|
https://github.com/holepunchto/hypercore-stats
|
|
785
|
-
hypercore-storage@3.
|
|
815
|
+
hypercore-storage@3.3.1
|
|
786
816
|
https://github.com/holepunchto/hypercore-storage
|
|
787
817
|
hyperdb@6.9.0
|
|
788
818
|
https://github.com/holepunchto/hyperdb
|
|
@@ -794,7 +824,7 @@ JavaScript Dependencies
|
|
|
794
824
|
https://github.com/holepunchto/hyperdispatch
|
|
795
825
|
hyperdrive@13.3.3
|
|
796
826
|
https://github.com/holepunchto/hyperdrive
|
|
797
|
-
hyperschema@1.
|
|
827
|
+
hyperschema@1.26.2
|
|
798
828
|
https://github.com/holepunchto/hyperschema
|
|
799
829
|
hyperswarm-stats@1.3.1
|
|
800
830
|
https://github.com/holepunchto/hyperswarm-stats
|
|
@@ -820,9 +850,11 @@ JavaScript Dependencies
|
|
|
820
850
|
https://github.com/holepunchto/refcounter
|
|
821
851
|
require-addon@1.3.0
|
|
822
852
|
https://github.com/holepunchto/require-addon
|
|
853
|
+
require-asset@1.2.4
|
|
854
|
+
https://github.com/holepunchto/require-asset
|
|
823
855
|
resource-on-exit@1.0.0
|
|
824
856
|
https://github.com/holepunchto/bare-teardown
|
|
825
|
-
rocksdb-native@3.
|
|
857
|
+
rocksdb-native@3.18.0
|
|
826
858
|
https://github.com/holepunchto/rocksdb-native
|
|
827
859
|
scope-lock@1.2.4
|
|
828
860
|
https://github.com/holepunchto/scope-lock
|
|
@@ -864,7 +896,7 @@ JavaScript Dependencies
|
|
|
864
896
|
https://github.com/mafintosh/bogon
|
|
865
897
|
codecs@3.1.0
|
|
866
898
|
https://github.com/mafintosh/codecs
|
|
867
|
-
corestore@7.12.
|
|
899
|
+
corestore@7.12.5
|
|
868
900
|
https://github.com/holepunchto/corestore
|
|
869
901
|
debounceify@1.1.0
|
|
870
902
|
https://github.com/mafintosh/debounceify
|
|
@@ -882,7 +914,7 @@ JavaScript Dependencies
|
|
|
882
914
|
https://github.com/mafintosh/generate-string
|
|
883
915
|
hyperbee@2.27.3
|
|
884
916
|
https://github.com/holepunchto/hyperbee
|
|
885
|
-
hypercore@11.
|
|
917
|
+
hypercore@11.36.1
|
|
886
918
|
https://github.com/holepunchto/hypercore
|
|
887
919
|
hypercore-crypto@3.7.0
|
|
888
920
|
https://github.com/mafintosh/hypercore-crypto
|
|
@@ -956,6 +988,6 @@ JavaScript Dependencies
|
|
|
956
988
|
https://github.com/mafintosh/xache
|
|
957
989
|
z32@1.1.0
|
|
958
990
|
https://github.com/mafintosh/z32
|
|
959
|
-
zod@4.6.
|
|
991
|
+
zod@4.6.5
|
|
960
992
|
https://github.com/colinhacks/zod
|
|
961
993
|
|
package/dist/api/audio-gen.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioGenClientParams, type AudioGenResult } from '../schemas/index.js';
|
|
1
|
+
import { type AudioEditClientParams, type AudioGenClientParams, type AudioGenResult, type AudioUnderstandClientParams, type AudioUnderstandResult } from '../schemas/index.js';
|
|
2
2
|
/**
|
|
3
3
|
* Generates audio (music / speech) from a caption using a loaded ACE-Step or
|
|
4
4
|
* MiniMax-Music3 AudioGen model.
|
|
@@ -16,3 +16,44 @@ import { type AudioGenClientParams, type AudioGenResult } from '../schemas/index
|
|
|
16
16
|
* ```
|
|
17
17
|
*/
|
|
18
18
|
export declare function audioGen(params: AudioGenClientParams): AudioGenResult;
|
|
19
|
+
/**
|
|
20
|
+
* Edits a source recording with a loaded ACE-Step AudioGen model: an ordered
|
|
21
|
+
* pipeline of `flow-edit` (re-condition the whole clip from one prompt to
|
|
22
|
+
* another) and `repaint` (regenerate a time range) operations, executed in
|
|
23
|
+
* array order. The source is interleaved stereo 48 kHz PCM — a file path
|
|
24
|
+
* decoded server-side, or raw Float32 LE bytes in `[-1, 1]`.
|
|
25
|
+
*
|
|
26
|
+
* @param params - Loaded model ID, the source audio, the ordered `operations`, and an optional `seed`.
|
|
27
|
+
* @returns The same run shape as `audioGen()`: `requestId`, `progressStream`, `audio`, `stats`, and `diagnostics`.
|
|
28
|
+
*
|
|
29
|
+
* @example
|
|
30
|
+
* ```typescript
|
|
31
|
+
* const run = audioEdit({
|
|
32
|
+
* modelId,
|
|
33
|
+
* sourceAudio: "/path/to/song.wav",
|
|
34
|
+
* operations: [
|
|
35
|
+
* { type: "flow-edit", from: { caption: "acoustic folk" }, to: { caption: "synthwave" } },
|
|
36
|
+
* { type: "repaint", caption: "analog synth solo", start: 10, end: 20 },
|
|
37
|
+
* ],
|
|
38
|
+
* seed: 7,
|
|
39
|
+
* });
|
|
40
|
+
* const { pcm, sampleRate, channels, bitsPerSample } = await run.audio;
|
|
41
|
+
* ```
|
|
42
|
+
*/
|
|
43
|
+
export declare function audioEdit(params: AudioEditClientParams): AudioGenResult;
|
|
44
|
+
/**
|
|
45
|
+
* Describes a recording with a loaded ACE-Step AudioGen model, running the
|
|
46
|
+
* engine's reverse pipeline: the PCM is encoded, its FSQ semantic codes are
|
|
47
|
+
* recovered, and the LM reports the clip's caption and metadata. The recovered
|
|
48
|
+
* `audioCodes` can be fed straight back into `audioGen()`.
|
|
49
|
+
*
|
|
50
|
+
* @param params - Loaded model ID, the source audio, and optional LM sampling controls.
|
|
51
|
+
* @returns `requestId`, `progressStream`, the `description`, `stats`, and `diagnostics`.
|
|
52
|
+
*
|
|
53
|
+
* @example
|
|
54
|
+
* ```typescript
|
|
55
|
+
* const run = audioUnderstand({ modelId, sourceAudio: "/path/to/song.wav" });
|
|
56
|
+
* const { caption, bpm, keyscale, audioCodes } = await run.description;
|
|
57
|
+
* ```
|
|
58
|
+
*/
|
|
59
|
+
export declare function audioUnderstand(params: AudioUnderstandClientParams): AudioUnderstandResult;
|
package/dist/api/audio-gen.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { audioGenClientParamsSchema, audioGenStreamResponseSchema } from '../schemas/index.js';
|
|
1
|
+
import { audioEditClientParamsSchema, audioEditStreamResponseSchema, audioGenClientParamsSchema, audioGenStreamResponseSchema, audioUnderstandClientParamsSchema, audioUnderstandResponseSchema } from '../schemas/index.js';
|
|
2
2
|
import { stream } from '../dispatch.js';
|
|
3
3
|
import { parseClientInput } from './parse-input.js';
|
|
4
4
|
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
@@ -38,21 +38,94 @@ export function audioGen(params) {
|
|
|
38
38
|
type: 'audioGenStream',
|
|
39
39
|
requestId
|
|
40
40
|
};
|
|
41
|
+
return collectAudioRun(request, requestId, audioGenStreamResponseSchema);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Edits a source recording with a loaded ACE-Step AudioGen model: an ordered
|
|
45
|
+
* pipeline of `flow-edit` (re-condition the whole clip from one prompt to
|
|
46
|
+
* another) and `repaint` (regenerate a time range) operations, executed in
|
|
47
|
+
* array order. The source is interleaved stereo 48 kHz PCM — a file path
|
|
48
|
+
* decoded server-side, or raw Float32 LE bytes in `[-1, 1]`.
|
|
49
|
+
*
|
|
50
|
+
* @param params - Loaded model ID, the source audio, the ordered `operations`, and an optional `seed`.
|
|
51
|
+
* @returns The same run shape as `audioGen()`: `requestId`, `progressStream`, `audio`, `stats`, and `diagnostics`.
|
|
52
|
+
*
|
|
53
|
+
* @example
|
|
54
|
+
* ```typescript
|
|
55
|
+
* const run = audioEdit({
|
|
56
|
+
* modelId,
|
|
57
|
+
* sourceAudio: "/path/to/song.wav",
|
|
58
|
+
* operations: [
|
|
59
|
+
* { type: "flow-edit", from: { caption: "acoustic folk" }, to: { caption: "synthwave" } },
|
|
60
|
+
* { type: "repaint", caption: "analog synth solo", start: 10, end: 20 },
|
|
61
|
+
* ],
|
|
62
|
+
* seed: 7,
|
|
63
|
+
* });
|
|
64
|
+
* const { pcm, sampleRate, channels, bitsPerSample } = await run.audio;
|
|
65
|
+
* ```
|
|
66
|
+
*/
|
|
67
|
+
export function audioEdit(params) {
|
|
68
|
+
const parsed = parseClientInput(audioEditClientParamsSchema, params);
|
|
69
|
+
const requestId = generateRandomRequestId();
|
|
70
|
+
const request = {
|
|
71
|
+
...parsed,
|
|
72
|
+
type: 'audioEditStream',
|
|
73
|
+
requestId
|
|
74
|
+
};
|
|
75
|
+
return collectAudioRun(request, requestId, audioEditStreamResponseSchema);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Describes a recording with a loaded ACE-Step AudioGen model, running the
|
|
79
|
+
* engine's reverse pipeline: the PCM is encoded, its FSQ semantic codes are
|
|
80
|
+
* recovered, and the LM reports the clip's caption and metadata. The recovered
|
|
81
|
+
* `audioCodes` can be fed straight back into `audioGen()`.
|
|
82
|
+
*
|
|
83
|
+
* @param params - Loaded model ID, the source audio, and optional LM sampling controls.
|
|
84
|
+
* @returns `requestId`, `progressStream`, the `description`, `stats`, and `diagnostics`.
|
|
85
|
+
*
|
|
86
|
+
* @example
|
|
87
|
+
* ```typescript
|
|
88
|
+
* const run = audioUnderstand({ modelId, sourceAudio: "/path/to/song.wav" });
|
|
89
|
+
* const { caption, bpm, keyscale, audioCodes } = await run.description;
|
|
90
|
+
* ```
|
|
91
|
+
*/
|
|
92
|
+
export function audioUnderstand(params) {
|
|
93
|
+
const parsed = parseClientInput(audioUnderstandClientParamsSchema, params);
|
|
94
|
+
const requestId = generateRandomRequestId();
|
|
95
|
+
const request = {
|
|
96
|
+
...parsed,
|
|
97
|
+
type: 'audioUnderstand',
|
|
98
|
+
requestId
|
|
99
|
+
};
|
|
100
|
+
let seen;
|
|
101
|
+
const { progressStream, payload, stats, diagnostics } = collectRun(request, requestId, audioUnderstandResponseSchema, {
|
|
102
|
+
absorb(frame) {
|
|
103
|
+
if (frame.understand !== undefined)
|
|
104
|
+
seen = frame.understand;
|
|
105
|
+
},
|
|
106
|
+
settle(frame) {
|
|
107
|
+
const result = frame.stats?.understand ?? seen;
|
|
108
|
+
if (result === undefined) {
|
|
109
|
+
throw new InvalidResponseError('audioUnderstand description');
|
|
110
|
+
}
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
return { requestId, progressStream, description: payload, stats, diagnostics };
|
|
115
|
+
}
|
|
116
|
+
function collectRun(request, requestId, responseSchema, sink) {
|
|
117
|
+
const wireType = request.type;
|
|
41
118
|
const progressQueue = [];
|
|
42
|
-
const pcmChunks = [];
|
|
43
|
-
let sampleRate;
|
|
44
|
-
let channels;
|
|
45
|
-
let bitsPerSample;
|
|
46
119
|
let progressDone = false;
|
|
47
120
|
let progressError;
|
|
48
121
|
let progressResolve;
|
|
49
|
-
let
|
|
50
|
-
let
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
|
|
122
|
+
let resolvePayload = () => { };
|
|
123
|
+
let rejectPayload = () => { };
|
|
124
|
+
const payload = new Promise((resolve, reject) => {
|
|
125
|
+
resolvePayload = resolve;
|
|
126
|
+
rejectPayload = reject;
|
|
54
127
|
});
|
|
55
|
-
|
|
128
|
+
payload.catch(() => { });
|
|
56
129
|
let resolveStats = () => { };
|
|
57
130
|
let rejectStats = () => { };
|
|
58
131
|
const stats = new Promise((resolve, reject) => {
|
|
@@ -71,6 +144,11 @@ export function audioGen(params) {
|
|
|
71
144
|
progressResolve?.();
|
|
72
145
|
progressResolve = undefined;
|
|
73
146
|
}
|
|
147
|
+
function rejectAll(error) {
|
|
148
|
+
rejectPayload(error);
|
|
149
|
+
rejectStats(error);
|
|
150
|
+
rejectDiagnostics(error);
|
|
151
|
+
}
|
|
74
152
|
async function processResponses() {
|
|
75
153
|
let receivedDone = false;
|
|
76
154
|
try {
|
|
@@ -78,53 +156,34 @@ export function audioGen(params) {
|
|
|
78
156
|
if (!response ||
|
|
79
157
|
typeof response !== 'object' ||
|
|
80
158
|
!('type' in response) ||
|
|
81
|
-
response.type !==
|
|
159
|
+
response.type !== wireType) {
|
|
82
160
|
continue;
|
|
83
161
|
}
|
|
84
|
-
const
|
|
85
|
-
if (
|
|
86
|
-
progressQueue.push(
|
|
162
|
+
const frame = responseSchema.parse(response);
|
|
163
|
+
if (frame.progress) {
|
|
164
|
+
progressQueue.push(frame.progress);
|
|
87
165
|
notifyProgress();
|
|
88
166
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
sampleRate = chunk.sampleRate;
|
|
92
|
-
channels = chunk.channels;
|
|
93
|
-
bitsPerSample = chunk.bitsPerSample;
|
|
94
|
-
}
|
|
95
|
-
if (chunk.done) {
|
|
167
|
+
sink.absorb(frame);
|
|
168
|
+
if (frame.done) {
|
|
96
169
|
receivedDone = true;
|
|
97
|
-
if (
|
|
98
|
-
|
|
99
|
-
rejectAudio(error);
|
|
100
|
-
rejectStats(error);
|
|
101
|
-
rejectDiagnostics(error);
|
|
170
|
+
if (frame.stopReason === 'cancelled') {
|
|
171
|
+
rejectAll(new InferenceCancelledError(requestId));
|
|
102
172
|
break;
|
|
103
173
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
resolveAudio({
|
|
108
|
-
pcm: concatenateChunks(pcmChunks),
|
|
109
|
-
sampleRate,
|
|
110
|
-
channels,
|
|
111
|
-
bitsPerSample
|
|
112
|
-
});
|
|
113
|
-
resolveStats(chunk.stats);
|
|
114
|
-
resolveDiagnostics(chunk.diagnostics);
|
|
174
|
+
resolvePayload(sink.settle(frame));
|
|
175
|
+
resolveStats(frame.stats);
|
|
176
|
+
resolveDiagnostics(frame.diagnostics);
|
|
115
177
|
break;
|
|
116
178
|
}
|
|
117
179
|
}
|
|
118
180
|
if (!receivedDone) {
|
|
119
|
-
throw new InvalidResponseError(
|
|
181
|
+
throw new InvalidResponseError(`${wireType} terminal response`);
|
|
120
182
|
}
|
|
121
183
|
}
|
|
122
184
|
catch (error) {
|
|
123
|
-
progressError =
|
|
124
|
-
|
|
125
|
-
rejectAudio(progressError);
|
|
126
|
-
rejectStats(progressError);
|
|
127
|
-
rejectDiagnostics(progressError);
|
|
185
|
+
progressError = error instanceof Error ? error : new InvalidResponseError(wireType, error);
|
|
186
|
+
rejectAll(progressError);
|
|
128
187
|
}
|
|
129
188
|
finally {
|
|
130
189
|
progressDone = true;
|
|
@@ -149,11 +208,38 @@ export function audioGen(params) {
|
|
|
149
208
|
}
|
|
150
209
|
}
|
|
151
210
|
void processResponses();
|
|
152
|
-
return {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
211
|
+
return { requestId, progressStream: progressStream(), payload, stats, diagnostics };
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Consumes one generation or editing stream: PCM chunks accumulate into
|
|
215
|
+
* `audio`, carrying the format reported alongside them.
|
|
216
|
+
*/
|
|
217
|
+
function collectAudioRun(request, requestId, responseSchema) {
|
|
218
|
+
const wireType = request.type;
|
|
219
|
+
const pcmChunks = [];
|
|
220
|
+
let sampleRate;
|
|
221
|
+
let channels;
|
|
222
|
+
let bitsPerSample;
|
|
223
|
+
const { progressStream, payload, stats, diagnostics } = collectRun(request, requestId, responseSchema, {
|
|
224
|
+
absorb(frame) {
|
|
225
|
+
if (frame.data === undefined)
|
|
226
|
+
return;
|
|
227
|
+
pcmChunks.push(decodeBase64(frame.data));
|
|
228
|
+
sampleRate = frame.sampleRate;
|
|
229
|
+
channels = frame.channels;
|
|
230
|
+
bitsPerSample = frame.bitsPerSample;
|
|
231
|
+
},
|
|
232
|
+
settle() {
|
|
233
|
+
if (sampleRate === undefined || channels === undefined || bitsPerSample === undefined) {
|
|
234
|
+
throw new InvalidResponseError(`${wireType} audio chunk`);
|
|
235
|
+
}
|
|
236
|
+
return {
|
|
237
|
+
pcm: concatenateChunks(pcmChunks),
|
|
238
|
+
sampleRate,
|
|
239
|
+
channels,
|
|
240
|
+
bitsPerSample
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
return { requestId, progressStream, audio: payload, stats, diagnostics };
|
|
159
245
|
}
|
package/dist/api/index.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export { finetune, type FinetuneHandle } from './finetune.js';
|
|
|
14
14
|
export { translate } from './translate.js';
|
|
15
15
|
export { cancel } from './cancel.js';
|
|
16
16
|
export { ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace } from './rag.js';
|
|
17
|
+
export { createVectorIndex, loadVectorIndex, type VectorIndex } from './vector-index.js';
|
|
17
18
|
export { textToSpeech, textToSpeechStream } from './text-to-speech.js';
|
|
18
19
|
export { getModelInfo } from './get-model-info.js';
|
|
19
20
|
export { getLoadedModelInfo } from './get-loaded-model-info.js';
|
|
@@ -22,7 +23,7 @@ export { assessModelFit } from './assess-model-fit.js';
|
|
|
22
23
|
export { ocr } from './ocr.js';
|
|
23
24
|
export { invokePlugin, invokePluginStream } from './invoke-plugin.js';
|
|
24
25
|
export { diffusion, type DiffusionProgressTick } from './diffusion.js';
|
|
25
|
-
export { audioGen } from './audio-gen.js';
|
|
26
|
+
export { audioGen, audioEdit, audioUnderstand } from './audio-gen.js';
|
|
26
27
|
export { classify } from './classify.js';
|
|
27
28
|
export { video, type VideoProgressTick } from './video.js';
|
|
28
29
|
export { upscale } from './upscale.js';
|
package/dist/api/index.js
CHANGED
|
@@ -14,6 +14,7 @@ export { finetune } from './finetune.js';
|
|
|
14
14
|
export { translate } from './translate.js';
|
|
15
15
|
export { cancel } from './cancel.js';
|
|
16
16
|
export { ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace } from './rag.js';
|
|
17
|
+
export { createVectorIndex, loadVectorIndex } from './vector-index.js';
|
|
17
18
|
export { textToSpeech, textToSpeechStream } from './text-to-speech.js';
|
|
18
19
|
export { getModelInfo } from './get-model-info.js';
|
|
19
20
|
export { getLoadedModelInfo } from './get-loaded-model-info.js';
|
|
@@ -22,7 +23,7 @@ export { assessModelFit } from './assess-model-fit.js';
|
|
|
22
23
|
export { ocr } from './ocr.js';
|
|
23
24
|
export { invokePlugin, invokePluginStream } from './invoke-plugin.js';
|
|
24
25
|
export { diffusion } from './diffusion.js';
|
|
25
|
-
export { audioGen } from './audio-gen.js';
|
|
26
|
+
export { audioGen, audioEdit, audioUnderstand } from './audio-gen.js';
|
|
26
27
|
export { classify } from './classify.js';
|
|
27
28
|
export { video } from './video.js';
|
|
28
29
|
export { upscale } from './upscale.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type TtsClientParamsInput, type RPCOptions, type TtsResponse, type TextToSpeechStreamClientParams, type TextToSpeechStreamSession, type TextToSpeechStreamResult } from '../schemas/index.js';
|
|
1
|
+
import { type TtsClientParamsInput, type RPCOptions, type TtsResponse, type TextToSpeechStreamClientParams, type TextToSpeechStreamSession, type TextToSpeechStreamResult, type TtsStats } from '../schemas/index.js';
|
|
2
2
|
/**
|
|
3
3
|
* Fan-out queue that lets multiple consumers iterate the same TTS response
|
|
4
4
|
* stream independently. Items are retained only until every active subscriber
|
|
@@ -33,9 +33,9 @@ export declare class TtsMulticast {
|
|
|
33
33
|
*
|
|
34
34
|
* Three modes selected by `params.stream` and `params.sentenceStream`:
|
|
35
35
|
*
|
|
36
|
-
* - `stream: false`
|
|
36
|
+
* - `stream: false` — collect all PCM samples and resolve once via
|
|
37
37
|
* `result.buffer` (`Promise<number[]>`). `bufferStream` is empty.
|
|
38
|
-
* - `stream: true` — yield PCM samples through `result.bufferStream`
|
|
38
|
+
* - `stream: true` (default) — yield PCM samples through `result.bufferStream`
|
|
39
39
|
* (`AsyncGenerator<number>`) as they arrive. `buffer` resolves to an empty
|
|
40
40
|
* array.
|
|
41
41
|
* - `stream: true, sentenceStream: true` — also exposes `result.chunkUpdates`
|
|
@@ -47,14 +47,41 @@ export declare class TtsMulticast {
|
|
|
47
47
|
* if the consumer breaks out before the terminal frame, or rejects on a
|
|
48
48
|
* pipeline error. Awaiting `done` is safe even when no stream is iterated.
|
|
49
49
|
*
|
|
50
|
+
* `result.sampleRate` resolves from the first audio frame (so it can be awaited
|
|
51
|
+
* before draining the audio); `result.stats` and `result.stopReason` resolve
|
|
52
|
+
* from the terminal frame. All three resolve — with `undefined` when the run
|
|
53
|
+
* produced none — rather than reject. In `stream: true` mode (the default) the
|
|
54
|
+
* run starts lazily: nothing is sent until `bufferStream` is first iterated, so
|
|
55
|
+
* `done`, `sampleRate`, `stats` and `stopReason` stay pending until then. In
|
|
56
|
+
* `stream: false` and `sentenceStream: true` modes the run starts immediately.
|
|
57
|
+
* Read `sampleRate` rather than assuming the engine default: `outputSampleRate`
|
|
58
|
+
* and the LavaSR enhancer both change it.
|
|
59
|
+
*
|
|
50
60
|
* @param params - TTS request parameters (see `TtsClientParamsInput`).
|
|
51
61
|
* @param options - Optional RPC options (timeout, profiling, force new connection).
|
|
52
62
|
* @returns A `TextToSpeechStreamResult` with `bufferStream`, `buffer`, `done`,
|
|
53
|
-
* and (when `sentenceStream: true`)
|
|
63
|
+
* `sampleRate`, `stats`, and (when `sentenceStream: true`)
|
|
64
|
+
* `chunkUpdates`.
|
|
54
65
|
* @throws {TextToSpeechStreamFailedError} When `sentenceStream: true` is paired
|
|
55
66
|
* with `stream: false`, or when the underlying RPC stream errors.
|
|
56
67
|
*/
|
|
57
68
|
export declare function textToSpeech(params: TtsClientParamsInput, options?: RPCOptions): TextToSpeechStreamResult;
|
|
69
|
+
/**
|
|
70
|
+
* The output rate and the runtime stats ride the response frames, but the
|
|
71
|
+
* three result shapes below expose PCM through generators a caller may abandon
|
|
72
|
+
* early. This collects both off the frames as they pass and hands them back as
|
|
73
|
+
* promises that always RESOLVE — with `undefined` when the run produced none —
|
|
74
|
+
* so `await result.sampleRate` can never hang or reject on its own.
|
|
75
|
+
*/
|
|
76
|
+
export type TtsSideChannel = {
|
|
77
|
+
sampleRate: Promise<number | undefined>;
|
|
78
|
+
stats: Promise<TtsStats | undefined>;
|
|
79
|
+
stopReason: Promise<'completed' | 'cancelled' | undefined>;
|
|
80
|
+
observe(response: TtsResponse): void;
|
|
81
|
+
settle(): void;
|
|
82
|
+
};
|
|
83
|
+
/** Exported for test use; not part of the public API surface. */
|
|
84
|
+
export declare function createTtsSideChannel(): TtsSideChannel;
|
|
58
85
|
/**
|
|
59
86
|
* Duplex session: write UTF-8 text fragments (e.g. LLM token deltas) via `write`. Each string or
|
|
60
87
|
* Buffer should be a complete UTF-8 fragment. The engine forwards them to ONNX TTS `runStreaming`
|