@qvac/inference 0.19.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/NOTICE +43 -11
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/index.d.ts +2 -1
  5. package/dist/api/index.js +2 -1
  6. package/dist/api/text-to-speech.d.ts +31 -4
  7. package/dist/api/text-to-speech.js +117 -21
  8. package/dist/api/vector-index.d.ts +142 -0
  9. package/dist/api/vector-index.js +168 -0
  10. package/dist/api/world.d.ts +2 -2
  11. package/dist/errors/index.d.ts +12 -0
  12. package/dist/errors/index.js +21 -0
  13. package/dist/handlers/vector-index.d.ts +2 -0
  14. package/dist/handlers/vector-index.js +40 -0
  15. package/dist/index.d.ts +1 -1
  16. package/dist/index.js +1 -1
  17. package/dist/models/registry/models.d.ts +351 -12
  18. package/dist/models/registry/models.js +2271 -1957
  19. package/dist/models/registry/resource-profiles.js +135 -0
  20. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  21. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  22. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  30. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  31. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  32. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  33. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  34. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  35. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  36. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  37. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  38. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  40. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  41. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  43. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  44. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  45. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
  46. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
  47. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  49. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
  50. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  51. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  52. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  53. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  54. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  55. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  56. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  57. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  58. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  59. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  60. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  61. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  62. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  63. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  64. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  65. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  66. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  67. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  68. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  70. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  71. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  72. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  73. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  74. package/dist/plugins/index.d.ts +1 -1
  75. package/dist/plugins/index.js +1 -1
  76. package/dist/plugins/ops/load-model.js +22 -2
  77. package/dist/plugins/ops/translate.js +58 -12
  78. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  79. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  80. package/dist/plugins/registry.d.ts +15 -0
  81. package/dist/plugins/registry.js +133 -15
  82. package/dist/plugins/turbovec-backend.d.ts +17 -0
  83. package/dist/plugins/turbovec-backend.js +99 -0
  84. package/dist/registry.js +4 -0
  85. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  86. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  87. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  88. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  89. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  90. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  91. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  92. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  93. package/dist/runtime/env.d.ts +2 -0
  94. package/dist/runtime/env.js +9 -2
  95. package/dist/runtime/lifecycle.js +2 -0
  96. package/dist/runtime/model-registry.d.ts +3 -1
  97. package/dist/runtime/model-registry.js +2 -1
  98. package/dist/runtime/request-context.js +17 -0
  99. package/dist/runtime/vector-index-backend.d.ts +25 -0
  100. package/dist/runtime/vector-index-backend.js +1 -0
  101. package/dist/runtime/vector-index-registry.d.ts +76 -0
  102. package/dist/runtime/vector-index-registry.js +161 -0
  103. package/dist/schemas/assess-model-fit.d.ts +44 -0
  104. package/dist/schemas/assess-model-fit.js +40 -0
  105. package/dist/schemas/audio-gen.d.ts +479 -4
  106. package/dist/schemas/audio-gen.js +376 -17
  107. package/dist/schemas/batch-completion-stream.d.ts +7 -0
  108. package/dist/schemas/batch-completion-stream.js +5 -3
  109. package/dist/schemas/common.d.ts +749 -15
  110. package/dist/schemas/common.js +8 -1
  111. package/dist/schemas/completion-event.d.ts +11 -1
  112. package/dist/schemas/completion-event.js +3 -0
  113. package/dist/schemas/completion-stream.d.ts +20 -0
  114. package/dist/schemas/completion-stream.js +41 -4
  115. package/dist/schemas/config.d.ts +48 -30
  116. package/dist/schemas/errors.d.ts +4 -0
  117. package/dist/schemas/errors.js +22 -0
  118. package/dist/schemas/index.d.ts +1 -0
  119. package/dist/schemas/index.js +1 -0
  120. package/dist/schemas/llamacpp-config.d.ts +4 -8
  121. package/dist/schemas/llamacpp-config.js +4 -4
  122. package/dist/schemas/load-model.d.ts +2503 -139
  123. package/dist/schemas/model-config-schemas.d.ts +228 -5
  124. package/dist/schemas/plugin.d.ts +10 -0
  125. package/dist/schemas/sdcpp-config.d.ts +221 -4
  126. package/dist/schemas/sdcpp-config.js +128 -22
  127. package/dist/schemas/text-to-speech.d.ts +780 -19
  128. package/dist/schemas/text-to-speech.js +298 -44
  129. package/dist/schemas/transcription-config.js +7 -4
  130. package/dist/schemas/vector-index.d.ts +194 -0
  131. package/dist/schemas/vector-index.js +185 -0
  132. package/dist/utils/addon-responses.d.ts +29 -3
  133. package/dist/utils/aggregate-events.d.ts +3 -1
  134. package/dist/utils/aggregate-events.js +7 -1
  135. package/dist/utils/tts-stats.d.ts +15 -11
  136. package/dist/utils/tts-stats.js +33 -11
  137. package/package.json +28 -25
package/NOTICE CHANGED
@@ -207,6 +207,18 @@ Third-Party Model Licenses
207
207
  https://huggingface.co/HuggingFaceVLA/smolvla_libero
208
208
  SmolVLM2-500M-Video-Instruct-GGUF
209
209
  https://huggingface.co/ggml-org/SmolVLM2-500M-Video-Instruct-GGUF
210
+ TranslatePsy-AfriSLM-0.8B-Q4-GGUF
211
+ https://huggingface.co/qvac/TranslatePsy-AfriSLM-0.8B-Q4-GGUF
212
+ TranslatePsy-AfriSLM-0.8B-Q8-GGUF
213
+ https://huggingface.co/qvac/TranslatePsy-AfriSLM-0.8B-Q8-GGUF
214
+ TranslatePsy-AfriSLM-2B-Q4-GGUF
215
+ https://huggingface.co/qvac/TranslatePsy-AfriSLM-2B-Q4-GGUF
216
+ TranslatePsy-AfriSLM-2B-Q8-GGUF
217
+ https://huggingface.co/qvac/TranslatePsy-AfriSLM-2B-Q8-GGUF
218
+ TranslatePsy-AfriSLM-4B-Q4-GGUF
219
+ https://huggingface.co/qvac/TranslatePsy-AfriSLM-4B-Q4-GGUF
220
+ TranslatePsy-AfriSLM-4B-Q8-GGUF
221
+ https://huggingface.co/qvac/TranslatePsy-AfriSLM-4B-Q8-GGUF
210
222
  umt5-xxl-enc-q8_0
211
223
  https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged
212
224
  unlimited-ocr-gguf
@@ -608,6 +620,12 @@ Third-Party Model Licenses
608
620
  https://huggingface.co/nvidia/GR00T-N1.7-LIBERO
609
621
  groot-q8_vf16
610
622
  https://huggingface.co/nvidia/GR00T-N1.7-3B
623
+ nemotron-3.5-asr-streaming-0.6b.f16
624
+ https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
625
+ nemotron-3.5-asr-streaming-0.6b.q4_0
626
+ https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
627
+ nemotron-3.5-asr-streaming-0.6b.q8_0
628
+ https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
611
629
  parakeet-unified-en-0.6b.f16
612
630
  https://huggingface.co/nvidia/parakeet-unified-en-0.6b
613
631
  parakeet-unified-en-0.6b.q4_0
@@ -663,11 +681,15 @@ JavaScript Dependencies
663
681
 
664
682
  --- apache-2.0 (Apache License 2.0) ---
665
683
 
666
- @hyperswarm/secret-stream@6.9.1
684
+ @hyperswarm/secret-stream@6.9.2
667
685
  https://github.com/holepunchto/hyperswarm-secret-stream
668
686
  @qvac/error@0.1.1
687
+ @qvac/fabric@0.16.0
688
+ https://github.com/tetherto/qvac
669
689
  @qvac/logging@0.1.1
670
690
  https://github.com/tetherto/qvac
691
+ @qvac/model-fit@0.12.1
692
+ https://github.com/tetherto/qvac
671
693
  @qvac/rag@0.8.1
672
694
  https://github.com/tetherto/qvac
673
695
  @qvac/registry-client@0.6.1
@@ -676,7 +698,7 @@ JavaScript Dependencies
676
698
  https://github.com/tetherto/qvac
677
699
  adaptive-timeout@1.0.1
678
700
  https://github.com/holepunchto/adaptive-timeout
679
- b4a@1.8.1
701
+ b4a@1.9.0
680
702
  https://github.com/holepunchto/b4a
681
703
  bare-abort@2.0.13
682
704
  https://github.com/holepunchto/bare-abort
@@ -694,13 +716,13 @@ JavaScript Dependencies
694
716
  https://github.com/holepunchto/bare-cpu-info
695
717
  bare-crypto@1.15.3
696
718
  https://github.com/holepunchto/bare-crypto
697
- bare-dns@2.2.0
719
+ bare-dns@2.2.1
698
720
  https://github.com/holepunchto/bare-dns
699
721
  bare-env@3.0.1
700
722
  https://github.com/holepunchto/bare-env
701
723
  bare-events@2.9.2
702
724
  https://github.com/holepunchto/bare-events
703
- bare-fetch@3.2.1
725
+ bare-fetch@3.4.0
704
726
  https://github.com/holepunchto/bare-fetch
705
727
  bare-form-data@1.2.2
706
728
  https://github.com/holepunchto/bare-form-data
@@ -712,7 +734,7 @@ JavaScript Dependencies
712
734
  https://github.com/holepunchto/bare-hrtime
713
735
  bare-http-parser@2.1.4
714
736
  https://github.com/holepunchto/bare-http-parser
715
- bare-http1@4.6.1
737
+ bare-http1@4.6.2
716
738
  https://github.com/holepunchto/bare-http1
717
739
  bare-https@3.1.0
718
740
  https://github.com/holepunchto/bare-https
@@ -738,6 +760,10 @@ JavaScript Dependencies
738
760
  https://github.com/holepunchto/bare-process
739
761
  bare-rpc@1.3.8
740
762
  https://github.com/holepunchto/bare-rpc
763
+ bare-runtime@1.33.4
764
+ https://github.com/holepunchto/bare-runtime
765
+ bare-runtime-darwin-arm64@1.33.4
766
+ https://github.com/holepunchto/bare-runtime
741
767
  bare-semver@1.1.0
742
768
  https://github.com/holepunchto/bare-semver
743
769
  bare-signals@5.0.0
@@ -746,6 +772,10 @@ JavaScript Dependencies
746
772
  https://github.com/holepunchto/bare-stdio
747
773
  bare-stream@2.13.4
748
774
  https://github.com/holepunchto/bare-stream
775
+ bare-structured-clone@1.6.0
776
+ https://github.com/holepunchto/bare-structured-clone
777
+ bare-subprocess@6.1.0
778
+ https://github.com/holepunchto/bare-subprocess
749
779
  bare-tcp@2.6.1
750
780
  https://github.com/holepunchto/bare-tcp
751
781
  bare-tls@3.1.10
@@ -782,7 +812,7 @@ JavaScript Dependencies
782
812
  https://github.com/holepunchto/hypercore-id-encoding
783
813
  hypercore-stats@2.5.1
784
814
  https://github.com/holepunchto/hypercore-stats
785
- hypercore-storage@3.2.1
815
+ hypercore-storage@3.3.1
786
816
  https://github.com/holepunchto/hypercore-storage
787
817
  hyperdb@6.9.0
788
818
  https://github.com/holepunchto/hyperdb
@@ -794,7 +824,7 @@ JavaScript Dependencies
794
824
  https://github.com/holepunchto/hyperdispatch
795
825
  hyperdrive@13.3.3
796
826
  https://github.com/holepunchto/hyperdrive
797
- hyperschema@1.24.0
827
+ hyperschema@1.26.2
798
828
  https://github.com/holepunchto/hyperschema
799
829
  hyperswarm-stats@1.3.1
800
830
  https://github.com/holepunchto/hyperswarm-stats
@@ -820,9 +850,11 @@ JavaScript Dependencies
820
850
  https://github.com/holepunchto/refcounter
821
851
  require-addon@1.3.0
822
852
  https://github.com/holepunchto/require-addon
853
+ require-asset@1.2.4
854
+ https://github.com/holepunchto/require-asset
823
855
  resource-on-exit@1.0.0
824
856
  https://github.com/holepunchto/bare-teardown
825
- rocksdb-native@3.17.4
857
+ rocksdb-native@3.18.0
826
858
  https://github.com/holepunchto/rocksdb-native
827
859
  scope-lock@1.2.4
828
860
  https://github.com/holepunchto/scope-lock
@@ -864,7 +896,7 @@ JavaScript Dependencies
864
896
  https://github.com/mafintosh/bogon
865
897
  codecs@3.1.0
866
898
  https://github.com/mafintosh/codecs
867
- corestore@7.12.4
899
+ corestore@7.12.5
868
900
  https://github.com/holepunchto/corestore
869
901
  debounceify@1.1.0
870
902
  https://github.com/mafintosh/debounceify
@@ -882,7 +914,7 @@ JavaScript Dependencies
882
914
  https://github.com/mafintosh/generate-string
883
915
  hyperbee@2.27.3
884
916
  https://github.com/holepunchto/hyperbee
885
- hypercore@11.35.5
917
+ hypercore@11.36.1
886
918
  https://github.com/holepunchto/hypercore
887
919
  hypercore-crypto@3.7.0
888
920
  https://github.com/mafintosh/hypercore-crypto
@@ -956,6 +988,6 @@ JavaScript Dependencies
956
988
  https://github.com/mafintosh/xache
957
989
  z32@1.1.0
958
990
  https://github.com/mafintosh/z32
959
- zod@4.6.1
991
+ zod@4.6.5
960
992
  https://github.com/colinhacks/zod
961
993
 
@@ -1,4 +1,4 @@
1
- import { type AudioGenClientParams, type AudioGenResult } from '../schemas/index.js';
1
+ import { type AudioEditClientParams, type AudioGenClientParams, type AudioGenResult, type AudioUnderstandClientParams, type AudioUnderstandResult } from '../schemas/index.js';
2
2
  /**
3
3
  * Generates audio (music / speech) from a caption using a loaded ACE-Step or
4
4
  * MiniMax-Music3 AudioGen model.
@@ -16,3 +16,44 @@ import { type AudioGenClientParams, type AudioGenResult } from '../schemas/index
16
16
  * ```
17
17
  */
18
18
  export declare function audioGen(params: AudioGenClientParams): AudioGenResult;
19
+ /**
20
+ * Edits a source recording with a loaded ACE-Step AudioGen model: an ordered
21
+ * pipeline of `flow-edit` (re-condition the whole clip from one prompt to
22
+ * another) and `repaint` (regenerate a time range) operations, executed in
23
+ * array order. The source is interleaved stereo 48 kHz PCM — a file path
24
+ * decoded server-side, or raw Float32 LE bytes in `[-1, 1]`.
25
+ *
26
+ * @param params - Loaded model ID, the source audio, the ordered `operations`, and an optional `seed`.
27
+ * @returns The same run shape as `audioGen()`: `requestId`, `progressStream`, `audio`, `stats`, and `diagnostics`.
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const run = audioEdit({
32
+ * modelId,
33
+ * sourceAudio: "/path/to/song.wav",
34
+ * operations: [
35
+ * { type: "flow-edit", from: { caption: "acoustic folk" }, to: { caption: "synthwave" } },
36
+ * { type: "repaint", caption: "analog synth solo", start: 10, end: 20 },
37
+ * ],
38
+ * seed: 7,
39
+ * });
40
+ * const { pcm, sampleRate, channels, bitsPerSample } = await run.audio;
41
+ * ```
42
+ */
43
+ export declare function audioEdit(params: AudioEditClientParams): AudioGenResult;
44
+ /**
45
+ * Describes a recording with a loaded ACE-Step AudioGen model, running the
46
+ * engine's reverse pipeline: the PCM is encoded, its FSQ semantic codes are
47
+ * recovered, and the LM reports the clip's caption and metadata. The recovered
48
+ * `audioCodes` can be fed straight back into `audioGen()`.
49
+ *
50
+ * @param params - Loaded model ID, the source audio, and optional LM sampling controls.
51
+ * @returns `requestId`, `progressStream`, the `description`, `stats`, and `diagnostics`.
52
+ *
53
+ * @example
54
+ * ```typescript
55
+ * const run = audioUnderstand({ modelId, sourceAudio: "/path/to/song.wav" });
56
+ * const { caption, bpm, keyscale, audioCodes } = await run.description;
57
+ * ```
58
+ */
59
+ export declare function audioUnderstand(params: AudioUnderstandClientParams): AudioUnderstandResult;
@@ -1,4 +1,4 @@
1
- import { audioGenClientParamsSchema, audioGenStreamResponseSchema } from '../schemas/index.js';
1
+ import { audioEditClientParamsSchema, audioEditStreamResponseSchema, audioGenClientParamsSchema, audioGenStreamResponseSchema, audioUnderstandClientParamsSchema, audioUnderstandResponseSchema } from '../schemas/index.js';
2
2
  import { stream } from '../dispatch.js';
3
3
  import { parseClientInput } from './parse-input.js';
4
4
  import { generateRandomRequestId } from '../runtime/request-id.js';
@@ -38,21 +38,94 @@ export function audioGen(params) {
38
38
  type: 'audioGenStream',
39
39
  requestId
40
40
  };
41
+ return collectAudioRun(request, requestId, audioGenStreamResponseSchema);
42
+ }
43
+ /**
44
+ * Edits a source recording with a loaded ACE-Step AudioGen model: an ordered
45
+ * pipeline of `flow-edit` (re-condition the whole clip from one prompt to
46
+ * another) and `repaint` (regenerate a time range) operations, executed in
47
+ * array order. The source is interleaved stereo 48 kHz PCM — a file path
48
+ * decoded server-side, or raw Float32 LE bytes in `[-1, 1]`.
49
+ *
50
+ * @param params - Loaded model ID, the source audio, the ordered `operations`, and an optional `seed`.
51
+ * @returns The same run shape as `audioGen()`: `requestId`, `progressStream`, `audio`, `stats`, and `diagnostics`.
52
+ *
53
+ * @example
54
+ * ```typescript
55
+ * const run = audioEdit({
56
+ * modelId,
57
+ * sourceAudio: "/path/to/song.wav",
58
+ * operations: [
59
+ * { type: "flow-edit", from: { caption: "acoustic folk" }, to: { caption: "synthwave" } },
60
+ * { type: "repaint", caption: "analog synth solo", start: 10, end: 20 },
61
+ * ],
62
+ * seed: 7,
63
+ * });
64
+ * const { pcm, sampleRate, channels, bitsPerSample } = await run.audio;
65
+ * ```
66
+ */
67
+ export function audioEdit(params) {
68
+ const parsed = parseClientInput(audioEditClientParamsSchema, params);
69
+ const requestId = generateRandomRequestId();
70
+ const request = {
71
+ ...parsed,
72
+ type: 'audioEditStream',
73
+ requestId
74
+ };
75
+ return collectAudioRun(request, requestId, audioEditStreamResponseSchema);
76
+ }
77
+ /**
78
+ * Describes a recording with a loaded ACE-Step AudioGen model, running the
79
+ * engine's reverse pipeline: the PCM is encoded, its FSQ semantic codes are
80
+ * recovered, and the LM reports the clip's caption and metadata. The recovered
81
+ * `audioCodes` can be fed straight back into `audioGen()`.
82
+ *
83
+ * @param params - Loaded model ID, the source audio, and optional LM sampling controls.
84
+ * @returns `requestId`, `progressStream`, the `description`, `stats`, and `diagnostics`.
85
+ *
86
+ * @example
87
+ * ```typescript
88
+ * const run = audioUnderstand({ modelId, sourceAudio: "/path/to/song.wav" });
89
+ * const { caption, bpm, keyscale, audioCodes } = await run.description;
90
+ * ```
91
+ */
92
+ export function audioUnderstand(params) {
93
+ const parsed = parseClientInput(audioUnderstandClientParamsSchema, params);
94
+ const requestId = generateRandomRequestId();
95
+ const request = {
96
+ ...parsed,
97
+ type: 'audioUnderstand',
98
+ requestId
99
+ };
100
+ let seen;
101
+ const { progressStream, payload, stats, diagnostics } = collectRun(request, requestId, audioUnderstandResponseSchema, {
102
+ absorb(frame) {
103
+ if (frame.understand !== undefined)
104
+ seen = frame.understand;
105
+ },
106
+ settle(frame) {
107
+ const result = frame.stats?.understand ?? seen;
108
+ if (result === undefined) {
109
+ throw new InvalidResponseError('audioUnderstand description');
110
+ }
111
+ return result;
112
+ }
113
+ });
114
+ return { requestId, progressStream, description: payload, stats, diagnostics };
115
+ }
116
+ function collectRun(request, requestId, responseSchema, sink) {
117
+ const wireType = request.type;
41
118
  const progressQueue = [];
42
- const pcmChunks = [];
43
- let sampleRate;
44
- let channels;
45
- let bitsPerSample;
46
119
  let progressDone = false;
47
120
  let progressError;
48
121
  let progressResolve;
49
- let resolveAudio = () => { };
50
- let rejectAudio = () => { };
51
- const audio = new Promise((resolve, reject) => {
52
- resolveAudio = resolve;
53
- rejectAudio = reject;
122
+ let resolvePayload = () => { };
123
+ let rejectPayload = () => { };
124
+ const payload = new Promise((resolve, reject) => {
125
+ resolvePayload = resolve;
126
+ rejectPayload = reject;
54
127
  });
55
- audio.catch(() => { });
128
+ payload.catch(() => { });
56
129
  let resolveStats = () => { };
57
130
  let rejectStats = () => { };
58
131
  const stats = new Promise((resolve, reject) => {
@@ -71,6 +144,11 @@ export function audioGen(params) {
71
144
  progressResolve?.();
72
145
  progressResolve = undefined;
73
146
  }
147
+ function rejectAll(error) {
148
+ rejectPayload(error);
149
+ rejectStats(error);
150
+ rejectDiagnostics(error);
151
+ }
74
152
  async function processResponses() {
75
153
  let receivedDone = false;
76
154
  try {
@@ -78,53 +156,34 @@ export function audioGen(params) {
78
156
  if (!response ||
79
157
  typeof response !== 'object' ||
80
158
  !('type' in response) ||
81
- response.type !== 'audioGenStream') {
159
+ response.type !== wireType) {
82
160
  continue;
83
161
  }
84
- const chunk = audioGenStreamResponseSchema.parse(response);
85
- if (chunk.progress) {
86
- progressQueue.push(chunk.progress);
162
+ const frame = responseSchema.parse(response);
163
+ if (frame.progress) {
164
+ progressQueue.push(frame.progress);
87
165
  notifyProgress();
88
166
  }
89
- if (chunk.data !== undefined) {
90
- pcmChunks.push(decodeBase64(chunk.data));
91
- sampleRate = chunk.sampleRate;
92
- channels = chunk.channels;
93
- bitsPerSample = chunk.bitsPerSample;
94
- }
95
- if (chunk.done) {
167
+ sink.absorb(frame);
168
+ if (frame.done) {
96
169
  receivedDone = true;
97
- if (chunk.stopReason === 'cancelled') {
98
- const error = new InferenceCancelledError(requestId);
99
- rejectAudio(error);
100
- rejectStats(error);
101
- rejectDiagnostics(error);
170
+ if (frame.stopReason === 'cancelled') {
171
+ rejectAll(new InferenceCancelledError(requestId));
102
172
  break;
103
173
  }
104
- if (sampleRate === undefined || channels === undefined || bitsPerSample === undefined) {
105
- throw new InvalidResponseError('audioGenStream audio chunk');
106
- }
107
- resolveAudio({
108
- pcm: concatenateChunks(pcmChunks),
109
- sampleRate,
110
- channels,
111
- bitsPerSample
112
- });
113
- resolveStats(chunk.stats);
114
- resolveDiagnostics(chunk.diagnostics);
174
+ resolvePayload(sink.settle(frame));
175
+ resolveStats(frame.stats);
176
+ resolveDiagnostics(frame.diagnostics);
115
177
  break;
116
178
  }
117
179
  }
118
180
  if (!receivedDone) {
119
- throw new InvalidResponseError('audioGenStream terminal response');
181
+ throw new InvalidResponseError(`${wireType} terminal response`);
120
182
  }
121
183
  }
122
184
  catch (error) {
123
- progressError =
124
- error instanceof Error ? error : new InvalidResponseError('audioGenStream', error);
125
- rejectAudio(progressError);
126
- rejectStats(progressError);
127
- rejectDiagnostics(progressError);
185
+ progressError = error instanceof Error ? error : new InvalidResponseError(wireType, error);
186
+ rejectAll(progressError);
128
187
  }
129
188
  finally {
130
189
  progressDone = true;
@@ -149,11 +208,38 @@ export function audioGen(params) {
149
208
  }
150
209
  }
151
210
  void processResponses();
152
- return {
153
- requestId,
154
- progressStream: progressStream(),
155
- audio,
156
- stats,
157
- diagnostics
158
- };
211
+ return { requestId, progressStream: progressStream(), payload, stats, diagnostics };
212
+ }
213
+ /**
214
+ * Consumes one generation or editing stream: PCM chunks accumulate into
215
+ * `audio`, carrying the format reported alongside them.
216
+ */
217
+ function collectAudioRun(request, requestId, responseSchema) {
218
+ const wireType = request.type;
219
+ const pcmChunks = [];
220
+ let sampleRate;
221
+ let channels;
222
+ let bitsPerSample;
223
+ const { progressStream, payload, stats, diagnostics } = collectRun(request, requestId, responseSchema, {
224
+ absorb(frame) {
225
+ if (frame.data === undefined)
226
+ return;
227
+ pcmChunks.push(decodeBase64(frame.data));
228
+ sampleRate = frame.sampleRate;
229
+ channels = frame.channels;
230
+ bitsPerSample = frame.bitsPerSample;
231
+ },
232
+ settle() {
233
+ if (sampleRate === undefined || channels === undefined || bitsPerSample === undefined) {
234
+ throw new InvalidResponseError(`${wireType} audio chunk`);
235
+ }
236
+ return {
237
+ pcm: concatenateChunks(pcmChunks),
238
+ sampleRate,
239
+ channels,
240
+ bitsPerSample
241
+ };
242
+ }
243
+ });
244
+ return { requestId, progressStream, audio: payload, stats, diagnostics };
159
245
  }
@@ -14,6 +14,7 @@ export { finetune, type FinetuneHandle } from './finetune.js';
14
14
  export { translate } from './translate.js';
15
15
  export { cancel } from './cancel.js';
16
16
  export { ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace } from './rag.js';
17
+ export { createVectorIndex, loadVectorIndex, type VectorIndex } from './vector-index.js';
17
18
  export { textToSpeech, textToSpeechStream } from './text-to-speech.js';
18
19
  export { getModelInfo } from './get-model-info.js';
19
20
  export { getLoadedModelInfo } from './get-loaded-model-info.js';
@@ -22,7 +23,7 @@ export { assessModelFit } from './assess-model-fit.js';
22
23
  export { ocr } from './ocr.js';
23
24
  export { invokePlugin, invokePluginStream } from './invoke-plugin.js';
24
25
  export { diffusion, type DiffusionProgressTick } from './diffusion.js';
25
- export { audioGen } from './audio-gen.js';
26
+ export { audioGen, audioEdit, audioUnderstand } from './audio-gen.js';
26
27
  export { classify } from './classify.js';
27
28
  export { video, type VideoProgressTick } from './video.js';
28
29
  export { upscale } from './upscale.js';
package/dist/api/index.js CHANGED
@@ -14,6 +14,7 @@ export { finetune } from './finetune.js';
14
14
  export { translate } from './translate.js';
15
15
  export { cancel } from './cancel.js';
16
16
  export { ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace } from './rag.js';
17
+ export { createVectorIndex, loadVectorIndex } from './vector-index.js';
17
18
  export { textToSpeech, textToSpeechStream } from './text-to-speech.js';
18
19
  export { getModelInfo } from './get-model-info.js';
19
20
  export { getLoadedModelInfo } from './get-loaded-model-info.js';
@@ -22,7 +23,7 @@ export { assessModelFit } from './assess-model-fit.js';
22
23
  export { ocr } from './ocr.js';
23
24
  export { invokePlugin, invokePluginStream } from './invoke-plugin.js';
24
25
  export { diffusion } from './diffusion.js';
25
- export { audioGen } from './audio-gen.js';
26
+ export { audioGen, audioEdit, audioUnderstand } from './audio-gen.js';
26
27
  export { classify } from './classify.js';
27
28
  export { video } from './video.js';
28
29
  export { upscale } from './upscale.js';
@@ -1,4 +1,4 @@
1
- import { type TtsClientParamsInput, type RPCOptions, type TtsResponse, type TextToSpeechStreamClientParams, type TextToSpeechStreamSession, type TextToSpeechStreamResult } from '../schemas/index.js';
1
+ import { type TtsClientParamsInput, type RPCOptions, type TtsResponse, type TextToSpeechStreamClientParams, type TextToSpeechStreamSession, type TextToSpeechStreamResult, type TtsStats } from '../schemas/index.js';
2
2
  /**
3
3
  * Fan-out queue that lets multiple consumers iterate the same TTS response
4
4
  * stream independently. Items are retained only until every active subscriber
@@ -33,9 +33,9 @@ export declare class TtsMulticast {
33
33
  *
34
34
  * Three modes selected by `params.stream` and `params.sentenceStream`:
35
35
  *
36
- * - `stream: false` (default) — collect all PCM samples and resolve once via
36
+ * - `stream: false` — collect all PCM samples and resolve once via
37
37
  * `result.buffer` (`Promise<number[]>`). `bufferStream` is empty.
38
- * - `stream: true` — yield PCM samples through `result.bufferStream`
38
+ * - `stream: true` (default) — yield PCM samples through `result.bufferStream`
39
39
  * (`AsyncGenerator<number>`) as they arrive. `buffer` resolves to an empty
40
40
  * array.
41
41
  * - `stream: true, sentenceStream: true` — also exposes `result.chunkUpdates`
@@ -47,14 +47,41 @@ export declare class TtsMulticast {
47
47
  * if the consumer breaks out before the terminal frame, or rejects on a
48
48
  * pipeline error. Awaiting `done` is safe even when no stream is iterated.
49
49
  *
50
+ * `result.sampleRate` resolves from the first audio frame (so it can be awaited
51
+ * before draining the audio); `result.stats` and `result.stopReason` resolve
52
+ * from the terminal frame. All three resolve — with `undefined` when the run
53
+ * produced none — rather than reject. In `stream: true` mode (the default) the
54
+ * run starts lazily: nothing is sent until `bufferStream` is first iterated, so
55
+ * `done`, `sampleRate`, `stats` and `stopReason` stay pending until then. In
56
+ * `stream: false` and `sentenceStream: true` modes the run starts immediately.
57
+ * Read `sampleRate` rather than assuming the engine default: `outputSampleRate`
58
+ * and the LavaSR enhancer both change it.
59
+ *
50
60
  * @param params - TTS request parameters (see `TtsClientParamsInput`).
51
61
  * @param options - Optional RPC options (timeout, profiling, force new connection).
52
62
  * @returns A `TextToSpeechStreamResult` with `bufferStream`, `buffer`, `done`,
53
- * and (when `sentenceStream: true`) `chunkUpdates`.
63
+ * `sampleRate`, `stats`, and (when `sentenceStream: true`)
64
+ * `chunkUpdates`.
54
65
  * @throws {TextToSpeechStreamFailedError} When `sentenceStream: true` is paired
55
66
  * with `stream: false`, or when the underlying RPC stream errors.
56
67
  */
57
68
  export declare function textToSpeech(params: TtsClientParamsInput, options?: RPCOptions): TextToSpeechStreamResult;
69
+ /**
70
+ * The output rate and the runtime stats ride the response frames, but the
71
+ * three result shapes below expose PCM through generators a caller may abandon
72
+ * early. This collects both off the frames as they pass and hands them back as
73
+ * promises that always RESOLVE — with `undefined` when the run produced none —
74
+ * so `await result.sampleRate` can never hang or reject on its own.
75
+ */
76
+ export type TtsSideChannel = {
77
+ sampleRate: Promise<number | undefined>;
78
+ stats: Promise<TtsStats | undefined>;
79
+ stopReason: Promise<'completed' | 'cancelled' | undefined>;
80
+ observe(response: TtsResponse): void;
81
+ settle(): void;
82
+ };
83
+ /** Exported for test use; not part of the public API surface. */
84
+ export declare function createTtsSideChannel(): TtsSideChannel;
58
85
  /**
59
86
  * Duplex session: write UTF-8 text fragments (e.g. LLM token deltas) via `write`. Each string or
60
87
  * Buffer should be a complete UTF-8 fragment. The engine forwards them to ONNX TTS `runStreaming`