@mentra/bluetooth-sdk 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -0,0 +1,1745 @@
1
+ // sherpa-onnx/c-api/cxx-api.h
2
+ //
3
+ // Copyright (c) 2024 Xiaomi Corporation
4
+ /**
5
+ * @file cxx-api.h
6
+ * @brief Public C++ wrapper for the sherpa-onnx C API.
7
+ *
8
+ * This header provides a lightweight C++ interface on top of `c-api.h`. The
9
+ * wrapper follows a few simple design rules:
10
+ *
11
+ * - Configuration objects are plain structs with `std::string`,
12
+ * `std::vector`, and default values
13
+ * - Runtime handles are move-only RAII classes that automatically release the
14
+ * underlying C handle
15
+ * - Result objects are copied into standard C++ containers so callers do not
16
+ * need to manage C-allocated memory manually
17
+ * - The API mirrors the C API closely, while offering a more idiomatic C++
18
+ * surface
19
+ *
20
+ * Major feature families available in this file:
21
+ *
22
+ * - Streaming ASR
23
+ * - Non-streaming ASR
24
+ * - Non-streaming TTS
25
+ * - Keyword spotting
26
+ * - Offline and online speech enhancement
27
+ * - VAD and circular buffering
28
+ * - Linear resampling
29
+ * - Version/file/WAVE helpers
30
+ * - Offline and online punctuation
31
+ * - Audio tagging
32
+ *
33
+ * Typical usage pattern:
34
+ *
35
+ * 1. Fill a config struct
36
+ * 2. Create the corresponding RAII wrapper with `Class::Create(...)`
37
+ * 3. Check `wrapper.Get()` for success
38
+ * 4. Feed audio or text, run inference, and retrieve results as C++ objects
39
+ * 5. Let destructors clean up automatically
40
+ *
41
+ * Example programs are available in `cxx-api-examples/` and show concrete model
42
+ * packages and end-to-end usage.
43
+ */
44
+ #ifndef SHERPA_ONNX_C_API_CXX_API_H_
45
+ #define SHERPA_ONNX_C_API_CXX_API_H_
46
+
47
+ #include <memory>
48
+ #include <string>
49
+ #include <unordered_map>
50
+ #include <vector>
51
+
52
+ #include "sherpa-onnx/c-api/c-api.h"
53
+
54
+ namespace sherpa_onnx::cxx {
55
+
56
+ // ============================================================================
57
+ // Streaming ASR
58
+ // ============================================================================
59
+ /** @brief Streaming transducer model files. */
60
+ struct OnlineTransducerModelConfig {
61
+ /** Encoder ONNX model. */
62
+ std::string encoder;
63
+ /** Decoder ONNX model. */
64
+ std::string decoder;
65
+ /** Joiner ONNX model. */
66
+ std::string joiner;
67
+ };
68
+
69
+ /** @brief Streaming Paraformer model files. */
70
+ struct OnlineParaformerModelConfig {
71
+ /** Encoder ONNX model. */
72
+ std::string encoder;
73
+ /** Decoder ONNX model. */
74
+ std::string decoder;
75
+ };
76
+
77
+ /** @brief Streaming Zipformer2 CTC model file. */
78
+ struct OnlineZipformer2CtcModelConfig {
79
+ /** Model ONNX file. */
80
+ std::string model;
81
+ };
82
+
83
+ /** @brief Streaming NeMo CTC model file. */
84
+ struct OnlineNemoCtcModelConfig {
85
+ /** Model ONNX file. */
86
+ std::string model;
87
+ };
88
+
89
+ /** @brief Streaming T-One CTC model file. */
90
+ struct OnlineToneCtcModelConfig {
91
+ /** Model ONNX file. */
92
+ std::string model;
93
+ };
94
+
95
+ /**
96
+ * @brief Acoustic model configuration for streaming ASR.
97
+ *
98
+ * Configure exactly one model family. If multiple model families are set, one
99
+ * of them will be chosen and the choice is implementation-defined.
100
+ *
101
+ * Example using
102
+ * `sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20`:
103
+ *
104
+ * @code
105
+ * OnlineModelConfig model;
106
+ * model.transducer.encoder =
107
+ * "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/"
108
+ * "encoder-epoch-99-avg-1.int8.onnx";
109
+ * model.transducer.decoder =
110
+ * "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/"
111
+ * "decoder-epoch-99-avg-1.onnx";
112
+ * model.transducer.joiner =
113
+ * "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/"
114
+ * "joiner-epoch-99-avg-1.int8.onnx";
115
+ * model.tokens =
116
+ * "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt";
117
+ * model.num_threads = 1;
118
+ * @endcode
119
+ */
120
+ struct OnlineModelConfig {
121
+ /** Streaming transducer configuration. */
122
+ OnlineTransducerModelConfig transducer;
123
+ /** Streaming Paraformer configuration. */
124
+ OnlineParaformerModelConfig paraformer;
125
+ /** Streaming Zipformer2 CTC configuration. */
126
+ OnlineZipformer2CtcModelConfig zipformer2_ctc;
127
+ /** Streaming NeMo CTC configuration. */
128
+ OnlineNemoCtcModelConfig nemo_ctc;
129
+ /** Streaming T-One CTC configuration. */
130
+ OnlineToneCtcModelConfig t_one_ctc;
131
+ /** Token file path. */
132
+ std::string tokens;
133
+ /** Number of inference threads. */
134
+ int32_t num_threads = 1;
135
+ /** Execution provider such as `"cpu"`. */
136
+ std::string provider = "cpu";
137
+ /** Enable verbose debug logging. */
138
+ bool debug = false;
139
+ /** Optional explicit model type hint. */
140
+ std::string model_type;
141
+ /** Modeling unit such as `"cjkchar"` or `"bpe"`. */
142
+ std::string modeling_unit = "cjkchar";
143
+ /** Optional BPE vocabulary. */
144
+ std::string bpe_vocab;
145
+ /** Optional in-memory token content. If non-empty, it is used instead of a
146
+ * file. */
147
+ std::string tokens_buf;
148
+ };
149
+
150
+ /** @brief Feature extraction settings shared by ASR and KWS wrappers. */
151
+ struct FeatureConfig {
152
+ /** Input sample rate in Hz. */
153
+ int32_t sample_rate = 16000;
154
+ /** Number of features per frame. */
155
+ int32_t feature_dim = 80;
156
+ };
157
+
158
+ /** @brief Decoder graph configuration for online CTC + FST decoding. */
159
+ struct OnlineCtcFstDecoderConfig {
160
+ /** FST graph file. */
161
+ std::string graph;
162
+ /** Maximum number of active states during search. */
163
+ int32_t max_active = 3000;
164
+ };
165
+
166
+ /** @brief Homophone replacement resources used by some Chinese ASR setups. */
167
+ struct HomophoneReplacerConfig {
168
+ /** Reserved field. Currently unused by the wrapper. */
169
+ std::string dict_dir;
170
+ /** Lexicon file used by the replacer. */
171
+ std::string lexicon;
172
+ /** Rule FST file used for replacement. */
173
+ std::string rule_fsts;
174
+ };
175
+
176
+ /**
177
+ * @brief Configuration for streaming ASR.
178
+ *
179
+ * Example:
180
+ *
181
+ * @code
182
+ * OnlineRecognizerConfig config;
183
+ * config.model_config.transducer.encoder =
184
+ * "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/"
185
+ * "encoder-epoch-99-avg-1.int8.onnx";
186
+ * config.model_config.transducer.decoder =
187
+ * "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/"
188
+ * "decoder-epoch-99-avg-1.onnx";
189
+ * config.model_config.transducer.joiner =
190
+ * "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/"
191
+ * "joiner-epoch-99-avg-1.int8.onnx";
192
+ * config.model_config.tokens =
193
+ * "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt";
194
+ * config.model_config.num_threads = 1;
195
+ * config.hr.lexicon = "./lexicon.txt";
196
+ * config.hr.rule_fsts = "./replace.fst";
197
+ * @endcode
198
+ */
199
+ struct OnlineRecognizerConfig {
200
+ /** Feature extraction configuration. */
201
+ FeatureConfig feat_config;
202
+ /** Acoustic model configuration. */
203
+ OnlineModelConfig model_config;
204
+
205
+ /** Decoding method such as `"greedy_search"` or `"modified_beam_search"`. */
206
+ std::string decoding_method = "greedy_search";
207
+
208
+ /** Maximum number of active paths for beam-search-style decoding. */
209
+ int32_t max_active_paths = 4;
210
+
211
+ /** Enable endpoint detection. */
212
+ bool enable_endpoint = false;
213
+
214
+ /** Endpointing rule 1 trailing silence threshold in seconds. */
215
+ float rule1_min_trailing_silence = 2.4;
216
+
217
+ /** Endpointing rule 2 trailing silence threshold in seconds. */
218
+ float rule2_min_trailing_silence = 1.2;
219
+
220
+ /** Endpointing rule 3 minimum utterance length in seconds. */
221
+ float rule3_min_utterance_length = 20;
222
+
223
+ /** Optional hotword file. */
224
+ std::string hotwords_file;
225
+
226
+ /** Hotword boost score. */
227
+ float hotwords_score = 1.5;
228
+
229
+ /** Optional CTC+FST decoder configuration. */
230
+ OnlineCtcFstDecoderConfig ctc_fst_decoder_config;
231
+ /** Optional ITN rule FST archive. */
232
+ std::string rule_fsts;
233
+ /** Optional ITN rule FAR archive. */
234
+ std::string rule_fars;
235
+ /** Optional blank penalty applied during decoding. */
236
+ float blank_penalty = 0;
237
+
238
+ /** Optional in-memory hotword definitions. */
239
+ std::string hotwords_buf;
240
+ /** Optional homophone replacement configuration. */
241
+ HomophoneReplacerConfig hr;
242
+ };
243
+
244
+ /** @brief Current streaming ASR result copied into C++ containers. */
245
+ struct OnlineRecognizerResult {
246
+ /** Decoded text. */
247
+ std::string text;
248
+ /** Token sequence. */
249
+ std::vector<std::string> tokens;
250
+ /** Per-token timestamps in seconds. */
251
+ std::vector<float> timestamps;
252
+ /** JSON representation of the result. */
253
+ std::string json;
254
+ };
255
+
256
+ /** @brief Mono PCM waveform used by the helper I/O functions. */
257
+ struct Wave {
258
+ /** Samples normalized to `[-1, 1]`. */
259
+ std::vector<float> samples;
260
+ /** Sample rate in Hz. */
261
+ int32_t sample_rate = 0;
262
+ };
263
+
264
+ /**
265
+ * @brief Read a mono WAVE file into a C++ value object.
266
+ *
267
+ * On failure, the returned wave has `samples.empty() == true`.
268
+ *
269
+ * @param filename Input WAVE filename.
270
+ * @return Decoded wave data.
271
+ */
272
+ SHERPA_ONNX_API Wave ReadWave(const std::string &filename);
273
+
274
+ /**
275
+ * @brief Write a mono WAVE file from a C++ value object.
276
+ *
277
+ * @param filename Output filename.
278
+ * @param wave PCM samples and sample rate to write.
279
+ * @return `true` on success; `false` on failure.
280
+ */
281
+ SHERPA_ONNX_API bool WriteWave(const std::string &filename, const Wave &wave);
282
+
283
+ /**
284
+ * @brief Base class for move-only RAII wrappers around C handles.
285
+ *
286
+ * Derived classes implement `Destroy(const T *) const` and inherit automatic
287
+ * destruction, `Get()`, and `Release()`.
288
+ */
289
+ template <typename Derived, typename T>
290
+ class SHERPA_ONNX_API MoveOnly {
291
+ public:
292
+ /** @brief Construct an empty wrapper. */
293
+ MoveOnly() = default;
294
+ /** @brief Construct a wrapper from a raw C handle. */
295
+ explicit MoveOnly(const T *p) : p_(p) {}
296
+
297
+ /** @brief Destroy the wrapped handle if present. */
298
+ ~MoveOnly() { Destroy(); }
299
+
300
+ MoveOnly(const MoveOnly &) = delete;
301
+
302
+ MoveOnly &operator=(const MoveOnly &) = delete;
303
+
304
+ MoveOnly(MoveOnly &&other) : p_(other.Release()) {}
305
+
306
+ MoveOnly &operator=(MoveOnly &&other) {
307
+ if (&other == this) {
308
+ return *this;
309
+ }
310
+
311
+ Destroy();
312
+
313
+ p_ = other.Release();
314
+
315
+ return *this;
316
+ }
317
+
318
+ /** @brief Return the wrapped raw pointer without transferring ownership. */
319
+ const T *Get() const { return p_; }
320
+
321
+ /** @brief Release ownership of the wrapped raw pointer. */
322
+ const T *Release() {
323
+ const T *p = p_;
324
+ p_ = nullptr;
325
+ return p;
326
+ }
327
+
328
+ private:
329
+ void Destroy() {
330
+ if (p_ == nullptr) {
331
+ return;
332
+ }
333
+
334
+ static_cast<Derived *>(this)->Destroy(p_);
335
+
336
+ p_ = nullptr;
337
+ }
338
+
339
+ protected:
340
+ const T *p_ = nullptr;
341
+ };
342
+
343
+ class SHERPA_ONNX_API OnlineStream
344
+ : public MoveOnly<OnlineStream, SherpaOnnxOnlineStream> {
345
+ public:
346
+ /** @brief Wrap an existing C online stream handle. */
347
+ explicit OnlineStream(const SherpaOnnxOnlineStream *p);
348
+
349
+ /** @brief Append audio samples to the stream. */
350
+ void AcceptWaveform(int32_t sample_rate, const float *samples,
351
+ int32_t n) const;
352
+
353
+ /** @brief Indicate that no more input audio will be provided. */
354
+ void InputFinished() const;
355
+
356
+ /** @brief Set a per-stream string option. */
357
+ void SetOption(const char *key, const char *value) const;
358
+ /** @brief Get a per-stream string option. */
359
+ const char *GetOption(const char *key) const;
360
+ /** @brief Check whether a per-stream option exists. */
361
+ int32_t HasOption(const char *key) const;
362
+
363
+ /** @brief Destroy the wrapped C handle. */
364
+ void Destroy(const SherpaOnnxOnlineStream *p) const;
365
+ };
366
+
367
+ /**
368
+ * @brief RAII wrapper for a streaming recognizer.
369
+ *
370
+ * Example:
371
+ *
372
+ * @code
373
+ * OnlineRecognizer recognizer = OnlineRecognizer::Create(config);
374
+ * OnlineStream stream = recognizer.CreateStream();
375
+ * stream.AcceptWaveform(wave.sample_rate, wave.samples.data(),
376
+ * wave.samples.size());
377
+ * stream.InputFinished();
378
+ * while (recognizer.IsReady(&stream)) {
379
+ * recognizer.Decode(&stream);
380
+ * }
381
+ * auto result = recognizer.GetResult(&stream);
382
+ * @endcode
383
+ */
384
+ class SHERPA_ONNX_API OnlineRecognizer
385
+ : public MoveOnly<OnlineRecognizer, SherpaOnnxOnlineRecognizer> {
386
+ public:
387
+ /** @brief Create a streaming recognizer from a config struct. */
388
+ static OnlineRecognizer Create(const OnlineRecognizerConfig &config);
389
+
390
+ /** @brief Destroy the wrapped C handle. */
391
+ void Destroy(const SherpaOnnxOnlineRecognizer *p) const;
392
+
393
+ /** @brief Create a stream that uses the recognizer's configured hotwords. */
394
+ OnlineStream CreateStream() const;
395
+
396
+ /** @brief Create a stream with inline hotwords. */
397
+ OnlineStream CreateStream(const std::string &hotwords) const;
398
+
399
+ /** @brief Check whether the given stream has enough data to decode. */
400
+ bool IsReady(const OnlineStream *s) const;
401
+
402
+ /** @brief Decode one ready stream. */
403
+ void Decode(const OnlineStream *s) const;
404
+
405
+ /** @brief Decode multiple ready streams in parallel. */
406
+ void Decode(const OnlineStream *ss, int32_t n) const;
407
+
408
+ /** @brief Return the current recognition result for a stream. */
409
+ OnlineRecognizerResult GetResult(const OnlineStream *s) const;
410
+
411
+ /** @brief Reset a stream after endpointing or utterance completion. */
412
+ void Reset(const OnlineStream *s) const;
413
+
414
+ /** @brief Check whether endpointing has triggered for a stream. */
415
+ bool IsEndpoint(const OnlineStream *s) const;
416
+
417
+ private:
418
+ explicit OnlineRecognizer(const SherpaOnnxOnlineRecognizer *p);
419
+ };
420
+
421
+ // ============================================================================
422
+ // Non-streaming ASR
423
+ // ============================================================================
424
+ /** @brief Offline transducer model files. */
425
+ struct OfflineTransducerModelConfig {
426
+ /** Encoder ONNX model. */
427
+ std::string encoder;
428
+ /** Decoder ONNX model. */
429
+ std::string decoder;
430
+ /** Joiner ONNX model. */
431
+ std::string joiner;
432
+ };
433
+
434
+ /** @brief Offline Paraformer model file. */
435
+ struct OfflineParaformerModelConfig {
436
+ /** Model ONNX file. */
437
+ std::string model;
438
+ };
439
+
440
+ /** @brief Offline NeMo EncDec CTC model file. */
441
+ struct OfflineNemoEncDecCtcModelConfig {
442
+ /** Model ONNX file. */
443
+ std::string model;
444
+ };
445
+
446
+ /** @brief Offline Whisper model configuration. */
447
+ struct OfflineWhisperModelConfig {
448
+ /** Encoder ONNX model. */
449
+ std::string encoder;
450
+ /** Decoder ONNX model. */
451
+ std::string decoder;
452
+ /** Whisper language string such as `"en"` or `"zh"`. */
453
+ std::string language;
454
+ /** Task such as `"transcribe"` or `"translate"`. */
455
+ std::string task = "transcribe";
456
+ /** Optional tail paddings in samples. */
457
+ int32_t tail_paddings = -1;
458
+ /** Enable token timestamps in the result. */
459
+ bool enable_token_timestamps = false;
460
+ /** Enable segment timestamps in the result JSON. */
461
+ bool enable_segment_timestamps = false;
462
+ };
463
+
464
+ /** @brief Offline Canary model configuration. */
465
+ struct OfflineCanaryModelConfig {
466
+ /** Encoder ONNX model. */
467
+ std::string encoder;
468
+ /** Decoder ONNX model. */
469
+ std::string decoder;
470
+ /** Source language code. */
471
+ std::string src_lang;
472
+ /** Target language code. */
473
+ std::string tgt_lang;
474
+ /** Whether punctuation/casing is enabled by the model. */
475
+ bool use_pnc = true;
476
+ };
477
+
478
+ /** @brief Offline Cohere Transcribe model configuration. */
479
+ struct OfflineCohereTranscribeModelConfig {
480
+ /** Encoder ONNX model. */
481
+ std::string encoder;
482
+ /** Decoder ONNX model. */
483
+ std::string decoder;
484
+ /** Cohere language string such as `"en"` or `"zh"`. */
485
+ std::string language;
486
+ /** Whether punctuation is enabled by the model. */
487
+ bool use_punct = true;
488
+ /** Whether inverse text normalization is enabled. */
489
+ bool use_itn = true;
490
+ };
491
+
492
+ /** @brief Offline FireRed ASR model files. */
493
+ struct OfflineFireRedAsrModelConfig {
494
+ /** Encoder ONNX model. */
495
+ std::string encoder;
496
+ /** Decoder ONNX model. */
497
+ std::string decoder;
498
+ };
499
+
500
+ /** @brief Offline FireRed ASR CTC model file. */
501
+ struct OfflineFireRedAsrCtcModelConfig {
502
+ /** Model ONNX file. */
503
+ std::string model;
504
+ };
505
+
506
+ /** @brief Offline TDNN model file. */
507
+ struct OfflineTdnnModelConfig {
508
+ /** Model ONNX file. */
509
+ std::string model;
510
+ };
511
+
512
+ /** @brief Offline SenseVoice model configuration. */
513
+ struct OfflineSenseVoiceModelConfig {
514
+ /** Model ONNX file. */
515
+ std::string model;
516
+ /** Language hint. */
517
+ std::string language;
518
+ /** Enable inverse text normalization. */
519
+ bool use_itn = false;
520
+ };
521
+
522
+ /** @brief Offline Dolphin model file. */
523
+ struct OfflineDolphinModelConfig {
524
+ /** Model ONNX file. */
525
+ std::string model;
526
+ };
527
+
528
+ /** @brief Offline Zipformer CTC model file. */
529
+ struct OfflineZipformerCtcModelConfig {
530
+ /** Model ONNX file. */
531
+ std::string model;
532
+ };
533
+
534
+ /** @brief Offline WeNet CTC model file. */
535
+ struct OfflineWenetCtcModelConfig {
536
+ /** Model ONNX file. */
537
+ std::string model;
538
+ };
539
+
540
+ /** @brief Offline omnilingual ASR CTC model file. */
541
+ struct OfflineOmnilingualAsrCtcModelConfig {
542
+ /** Model ONNX file. */
543
+ std::string model;
544
+ };
545
+
546
+ /** @brief Offline MedASR CTC model file. */
547
+ struct OfflineMedAsrCtcModelConfig {
548
+ /** Model ONNX file. */
549
+ std::string model;
550
+ };
551
+
552
+ /** @brief Offline Moonshine model configuration. */
553
+ struct OfflineMoonshineModelConfig {
554
+ /** Preprocessor model file. */
555
+ std::string preprocessor;
556
+ /** Encoder model file. */
557
+ std::string encoder;
558
+ /** Uncached decoder model file. */
559
+ std::string uncached_decoder;
560
+ /** Cached decoder model file. */
561
+ std::string cached_decoder;
562
+ /** Merged decoder model file. */
563
+ std::string merged_decoder;
564
+ };
565
+
566
+ /** @brief Offline FunASR Nano model configuration. */
567
+ struct OfflineFunASRNanoModelConfig {
568
+ /** Encoder adaptor model file. */
569
+ std::string encoder_adaptor;
570
+ /** LLM model file. */
571
+ std::string llm;
572
+ /** Embedding model file. */
573
+ std::string embedding;
574
+ /** Tokenizer file. */
575
+ std::string tokenizer;
576
+ /** System prompt passed to the model. */
577
+ std::string system_prompt = "You are a helpful assistant.";
578
+ /** User prompt prefix passed to the model. */
579
+ std::string user_prompt = "语音转写:";
580
+ /** Maximum number of generated tokens. */
581
+ int32_t max_new_tokens = 512;
582
+ /** Sampling temperature. */
583
+ float temperature = 1e-6f;
584
+ /** Top-p sampling parameter. */
585
+ float top_p = 0.8f;
586
+ /** Random seed. */
587
+ int32_t seed = 42;
588
+ /** Language hint. */
589
+ std::string language;
590
+ /** Enable inverse text normalization. */
591
+ bool itn = true;
592
+ /** Optional hotwords string. */
593
+ std::string hotwords;
594
+ };
595
+
596
+ /** @brief Offline Qwen3-ASR model configuration. */
597
+ struct OfflineQwen3ASRModelConfig {
598
+ /** Conv-frontend ONNX model file. */
599
+ std::string conv_frontend;
600
+ /** Encoder ONNX model file. */
601
+ std::string encoder;
602
+ /** Decoder ONNX model file (KV cache). */
603
+ std::string decoder;
604
+ /** Tokenizer directory (e.g. containing `vocab.json`). */
605
+ std::string tokenizer;
606
+ /** Optional comma-separated hotwords (UTF-8, ASCII ','), e.g. @c
607
+ * "foo,bar,baz". */
608
+ std::string hotwords;
609
+ /** Maximum total sequence length supported by the model. */
610
+ int32_t max_total_len = 512;
611
+ /** Maximum number of new tokens to generate. */
612
+ int32_t max_new_tokens = 128;
613
+ /** Sampling temperature. */
614
+ float temperature = 1e-6f;
615
+ /** Top-p (nucleus) sampling parameter. */
616
+ float top_p = 0.8f;
617
+ /** Random seed for reproducible sampling. */
618
+ int32_t seed = 42;
619
+ };
620
+
621
+ /**
622
+ * @brief Acoustic model configuration for offline ASR.
623
+ *
624
+ * Configure exactly one model family. If multiple model families are set, one
625
+ * is chosen and the choice is implementation-defined.
626
+ */
627
+ struct OfflineModelConfig {
628
+ /** Offline transducer configuration. */
629
+ OfflineTransducerModelConfig transducer;
630
+ /** Offline Paraformer configuration. */
631
+ OfflineParaformerModelConfig paraformer;
632
+ /** Offline NeMo CTC configuration. */
633
+ OfflineNemoEncDecCtcModelConfig nemo_ctc;
634
+ /** Offline Whisper configuration. */
635
+ OfflineWhisperModelConfig whisper;
636
+ /** Offline TDNN configuration. */
637
+ OfflineTdnnModelConfig tdnn;
638
+
639
+ /** Token file. */
640
+ std::string tokens;
641
+ /** Number of inference threads. */
642
+ int32_t num_threads = 1;
643
+ /** Enable verbose debug logging. */
644
+ bool debug = false;
645
+ /** Execution provider such as `"cpu"`. */
646
+ std::string provider = "cpu";
647
+ /** Optional explicit model type hint. */
648
+ std::string model_type;
649
+ /** Modeling unit such as `"cjkchar"` or `"bpe"`. */
650
+ std::string modeling_unit = "cjkchar";
651
+ /** Optional BPE vocabulary. */
652
+ std::string bpe_vocab;
653
+ /** Telespeech CTC model file. */
654
+ std::string telespeech_ctc;
655
+ /** SenseVoice configuration. */
656
+ OfflineSenseVoiceModelConfig sense_voice;
657
+ /** Moonshine configuration. */
658
+ OfflineMoonshineModelConfig moonshine;
659
+ /** FireRed transducer configuration. */
660
+ OfflineFireRedAsrModelConfig fire_red_asr;
661
+ /** Dolphin configuration. */
662
+ OfflineDolphinModelConfig dolphin;
663
+ /** Zipformer CTC configuration. */
664
+ OfflineZipformerCtcModelConfig zipformer_ctc;
665
+ /** Canary configuration. */
666
+ OfflineCanaryModelConfig canary;
667
+ /** WeNet CTC configuration. */
668
+ OfflineWenetCtcModelConfig wenet_ctc;
669
+ /** Omnilingual ASR configuration. */
670
+ OfflineOmnilingualAsrCtcModelConfig omnilingual;
671
+ /** MedASR configuration. */
672
+ OfflineMedAsrCtcModelConfig medasr;
673
+ /** FunASR Nano configuration. */
674
+ OfflineFunASRNanoModelConfig funasr_nano;
675
+ /** FireRed CTC configuration. */
676
+ OfflineFireRedAsrCtcModelConfig fire_red_asr_ctc;
677
+ /** Qwen3-ASR configuration. */
678
+ OfflineQwen3ASRModelConfig qwen3_asr;
679
+ /** Cohere Transcribe configuration. */
680
+ OfflineCohereTranscribeModelConfig cohere_transcribe;
681
+ };
682
+
683
+ /** @brief Optional language-model rescoring configuration for offline ASR. */
684
+ struct OfflineLMConfig {
685
+ /** LM model file. */
686
+ std::string model;
687
+ /** LM scale. */
688
+ float scale = 1.0;
689
+ };
690
+
691
+ /**
692
+ * @brief Configuration for offline ASR.
693
+ *
694
+ * Example using SenseVoice:
695
+ *
696
+ * @code
697
+ * OfflineRecognizerConfig config;
698
+ * config.model_config.sense_voice.model =
699
+ * "./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8/model.int8.onnx";
700
+ * config.model_config.sense_voice.language = "auto";
701
+ * config.model_config.sense_voice.use_itn = true;
702
+ * config.model_config.tokens =
703
+ * "./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8/tokens.txt";
704
+ * config.model_config.num_threads = 1;
705
+ * @endcode
706
+ *
707
+ * Example using Parakeet TDT v2:
708
+ *
709
+ * @code
710
+ * OfflineRecognizerConfig config;
711
+ * config.model_config.transducer.encoder =
712
+ * "./sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8/encoder.int8.onnx";
713
+ * config.model_config.transducer.decoder =
714
+ * "./sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8/decoder.int8.onnx";
715
+ * config.model_config.transducer.joiner =
716
+ * "./sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8/joiner.int8.onnx";
717
+ * config.model_config.tokens =
718
+ * "./sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8/tokens.txt";
719
+ * config.model_config.model_type = "nemo_transducer";
720
+ * config.model_config.num_threads = 1;
721
+ * @endcode
722
+ */
723
+ struct OfflineRecognizerConfig {
724
+ /** Feature extraction configuration. */
725
+ FeatureConfig feat_config;
726
+ /** Acoustic model configuration. */
727
+ OfflineModelConfig model_config;
728
+ /** Optional LM configuration. */
729
+ OfflineLMConfig lm_config;
730
+
731
+ /** Decoding method such as `"greedy_search"` or `"modified_beam_search"`. */
732
+ std::string decoding_method = "greedy_search";
733
+ /** Maximum number of active paths for beam-search-style decoding. */
734
+ int32_t max_active_paths = 4;
735
+
736
+ /** Optional hotword file. */
737
+ std::string hotwords_file;
738
+
739
+ /** Hotword boost score. */
740
+ float hotwords_score = 1.5;
741
+ /** Optional ITN rule FST archive. */
742
+ std::string rule_fsts;
743
+ /** Optional ITN rule FAR archive. */
744
+ std::string rule_fars;
745
+ /** Optional blank penalty applied during decoding. */
746
+ float blank_penalty = 0;
747
+ /** Optional homophone replacement configuration. */
748
+ HomophoneReplacerConfig hr;
749
+ };
750
+
751
+ /** @brief Offline ASR result copied into C++ containers. */
752
+ struct OfflineRecognizerResult {
753
+ /** Decoded text. */
754
+ std::string text;
755
+ /** Per-token timestamps in seconds when available. */
756
+ std::vector<float> timestamps;
757
+ /** Token sequence. */
758
+ std::vector<std::string> tokens;
759
+ /** JSON representation of the result. */
760
+ std::string json;
761
+ /** Detected language when provided by the model. */
762
+ std::string lang;
763
+ /** Detected emotion when provided by the model. */
764
+ std::string emotion;
765
+ /** Detected event when provided by the model. */
766
+ std::string event;
767
+
768
+ /** Non-empty only for TDT-style models. */
769
+ std::vector<float> durations;
770
+ };
771
+
772
+ /** @brief RAII wrapper for an offline decoding stream. */
773
+ class SHERPA_ONNX_API OfflineStream
774
+ : public MoveOnly<OfflineStream, SherpaOnnxOfflineStream> {
775
+ public:
776
+ /** @brief Wrap an existing C offline stream handle. */
777
+ explicit OfflineStream(const SherpaOnnxOfflineStream *p);
778
+
779
+ /** @brief Provide the complete waveform for offline decoding. */
780
+ void AcceptWaveform(int32_t sample_rate, const float *samples,
781
+ int32_t n) const;
782
+
783
+ /** @brief Set a per-stream string option. */
784
+ void SetOption(const char *key, const char *value) const;
785
+ /** @brief Get a per-stream string option. */
786
+ const char *GetOption(const char *key) const;
787
+ /** @brief Check whether a per-stream option exists. */
788
+ int32_t HasOption(const char *key) const;
789
+
790
+ /** @brief Destroy the wrapped C handle. */
791
+ void Destroy(const SherpaOnnxOfflineStream *p) const;
792
+ };
793
+
794
+ /**
795
+ * @brief RAII wrapper for an offline recognizer.
796
+ *
797
+ * For most offline models, call `AcceptWaveform()` once per stream, then call
798
+ * `Decode()` and `GetResult()`.
799
+ */
800
+ class SHERPA_ONNX_API OfflineRecognizer
801
+ : public MoveOnly<OfflineRecognizer, SherpaOnnxOfflineRecognizer> {
802
+ public:
803
+ /** @brief Create an offline recognizer from a config struct. */
804
+ static OfflineRecognizer Create(const OfflineRecognizerConfig &config);
805
+
806
+ /** @brief Destroy the wrapped C handle. */
807
+ void Destroy(const SherpaOnnxOfflineRecognizer *p) const;
808
+
809
+ /** @brief Create a stream using the recognizer's configured hotwords. */
810
+ OfflineStream CreateStream() const;
811
+
812
+ /** @brief Create a stream with inline hotwords. */
813
+ OfflineStream CreateStream(const std::string &hotwords) const;
814
+
815
+ /** @brief Decode one offline stream. */
816
+ void Decode(const OfflineStream *s) const;
817
+
818
+ /** @brief Decode multiple offline streams in parallel. */
819
+ void Decode(const OfflineStream *ss, int32_t n) const;
820
+
821
+ /** @brief Return the copied recognition result for one stream. */
822
+ OfflineRecognizerResult GetResult(const OfflineStream *s) const;
823
+
824
+ /**
825
+ * @brief Convenience wrapper that returns the result inside a shared pointer.
826
+ *
827
+ * This helper exists mainly for integration environments that prefer owning
828
+ * pointers, such as Unreal Engine.
829
+ */
830
+ std::shared_ptr<OfflineRecognizerResult> GetResultPtr(
831
+ const OfflineStream *s) const;
832
+
833
+ /** @brief Update recognizer runtime configuration after creation. */
834
+ void SetConfig(const OfflineRecognizerConfig &config) const;
835
+
836
+ private:
837
+ explicit OfflineRecognizer(const SherpaOnnxOfflineRecognizer *p);
838
+ };
839
+
840
+ // ============================================================================
841
+ // Non-streaming TTS
842
+ // ============================================================================
843
+ /** @brief VITS model configuration. */
844
+ struct OfflineTtsVitsModelConfig {
845
+ /** Acoustic model file. */
846
+ std::string model;
847
+ /** Lexicon file. */
848
+ std::string lexicon;
849
+ /** Token file. */
850
+ std::string tokens;
851
+ /** Data directory such as `espeak-ng-data`. */
852
+ std::string data_dir;
853
+ /** Reserved field. Currently unused by the wrapper. */
854
+ std::string dict_dir;
855
+
856
+ /** VITS noise scale. */
857
+ float noise_scale = 0.667;
858
+ /** VITS noise scale for duration prediction. */
859
+ float noise_scale_w = 0.8;
860
+ /** Length scale. Values < 1 are faster; values > 1 are slower. */
861
+ float length_scale = 1.0;
862
+ };
863
+
864
+ /** @brief Matcha model configuration. */
865
+ struct OfflineTtsMatchaModelConfig {
866
+ /** Acoustic model file. */
867
+ std::string acoustic_model;
868
+ /** Vocoder model file. */
869
+ std::string vocoder;
870
+ /** Lexicon file. */
871
+ std::string lexicon;
872
+ /** Token file. */
873
+ std::string tokens;
874
+ /** Data directory such as `espeak-ng-data`. */
875
+ std::string data_dir;
876
+ /** Reserved field. Currently unused by the wrapper. */
877
+ std::string dict_dir;
878
+
879
+ /** Matcha noise scale. */
880
+ float noise_scale = 0.667;
881
+ /** Length scale. Values < 1 are faster; values > 1 are slower. */
882
+ float length_scale = 1.0;
883
+ };
884
+
885
+ /** @brief Kokoro model configuration. */
886
+ struct OfflineTtsKokoroModelConfig {
887
+ /** Acoustic model file. */
888
+ std::string model;
889
+ /** Voices file. */
890
+ std::string voices;
891
+ /** Token file. */
892
+ std::string tokens;
893
+ /** Data directory such as `espeak-ng-data`. */
894
+ std::string data_dir;
895
+ /** Reserved field. Currently unused by the wrapper. */
896
+ std::string dict_dir;
897
+ /** Optional lexicon file. */
898
+ std::string lexicon;
899
+ /** Language/voice family hint. */
900
+ std::string lang;
901
+
902
+ /** Length scale. Values < 1 are faster; values > 1 are slower. */
903
+ float length_scale = 1.0;
904
+ };
905
+
906
+ /** @brief Kitten model configuration. */
907
+ struct OfflineTtsKittenModelConfig {
908
+ /** Acoustic model file. */
909
+ std::string model;
910
+ /** Voices file. */
911
+ std::string voices;
912
+ /** Token file. */
913
+ std::string tokens;
914
+ /** Data directory. */
915
+ std::string data_dir;
916
+
917
+ /** Length scale. Values < 1 are faster; values > 1 are slower. */
918
+ float length_scale = 1.0;
919
+ };
920
+
921
+ /** @brief ZipVoice model configuration. */
922
+ struct OfflineTtsZipvoiceModelConfig {
923
+ /** Token file. */
924
+ std::string tokens;
925
+ /** Encoder model file. */
926
+ std::string encoder;
927
+ /** Decoder model file. */
928
+ std::string decoder;
929
+ /** Vocoder model file. */
930
+ std::string vocoder;
931
+ /** Data directory. */
932
+ std::string data_dir;
933
+ /** Lexicon file. */
934
+ std::string lexicon;
935
+
936
+ /** Feature scale. */
937
+ float feat_scale = 0.1;
938
+ /** Time shift. */
939
+ float t_shift = 0.5;
940
+ /** Target RMS. */
941
+ float target_rms = 0.1;
942
+ /** Guidance scale. */
943
+ float guidance_scale = 1.0;
944
+ };
945
+
946
+ /** @brief Pocket TTS model configuration. */
947
+ struct OfflineTtsPocketModelConfig {
948
+ /** Flow model file. */
949
+ std::string lm_flow;
950
+ /** Main language model file. */
951
+ std::string lm_main;
952
+ /** Encoder model file. */
953
+ std::string encoder;
954
+ /** Decoder model file. */
955
+ std::string decoder;
956
+ /** Text conditioner model file. */
957
+ std::string text_conditioner;
958
+
959
+ /** Vocabulary JSON file. */
960
+ std::string vocab_json;
961
+ /** Token scores JSON file. */
962
+ std::string token_scores_json;
963
+ /** Voice embedding cache size. */
964
+ int32_t voice_embedding_cache_capacity = 50;
965
+ };
966
+
967
+ /** @brief Supertonic model configuration. */
968
+ struct OfflineTtsSupertonicModelConfig {
969
+ /** Duration predictor model file. */
970
+ std::string duration_predictor;
971
+ /** Text encoder model file. */
972
+ std::string text_encoder;
973
+ /** Vector estimator model file. */
974
+ std::string vector_estimator;
975
+ /** Vocoder model file. */
976
+ std::string vocoder;
977
+ /** Model metadata JSON. */
978
+ std::string tts_json;
979
+ /** Unicode indexer resource. */
980
+ std::string unicode_indexer;
981
+ /** Voice style resource. */
982
+ std::string voice_style;
983
+ };
984
+
985
+ /**
986
+ * @brief Model configuration for offline TTS.
987
+ *
988
+ * Configure exactly one model family. If multiple model families are set, one
989
+ * is chosen and the choice is implementation-defined.
990
+ */
991
+ struct OfflineTtsModelConfig {
992
+ /** VITS configuration. */
993
+ OfflineTtsVitsModelConfig vits;
994
+ /** Matcha configuration. */
995
+ OfflineTtsMatchaModelConfig matcha;
996
+ /** Kokoro configuration. */
997
+ OfflineTtsKokoroModelConfig kokoro;
998
+ /** Kitten configuration. */
999
+ OfflineTtsKittenModelConfig kitten;
1000
+ /** ZipVoice configuration. */
1001
+ OfflineTtsZipvoiceModelConfig zipvoice;
1002
+ /** Pocket configuration. */
1003
+ OfflineTtsPocketModelConfig pocket;
1004
+ /** Supertonic configuration. */
1005
+ OfflineTtsSupertonicModelConfig supertonic;
1006
+
1007
+ /** Number of inference threads. */
1008
+ int32_t num_threads = 1;
1009
+ /** Enable verbose debug logging. */
1010
+ bool debug = false;
1011
+ /** Execution provider such as `"cpu"`. */
1012
+ std::string provider = "cpu";
1013
+ };
1014
+
1015
+ /** @brief Generation-time options for advanced TTS synthesis. */
1016
+ struct GenerationConfig {
1017
+ /** Silence scale between sentences. */
1018
+ float silence_scale = 0.2;
1019
+ /** Speech speed. Used only by some models. */
1020
+ float speed = 1.0;
1021
+ /** Speaker ID for multi-speaker models. */
1022
+ int32_t sid = 0;
1023
+ /** Reference audio samples for zero-shot or voice-cloning models. */
1024
+ std::vector<float> reference_audio;
1025
+ /** Sample rate of `reference_audio`. */
1026
+ int32_t reference_sample_rate = 0;
1027
+ /** Optional reference text. Not all models require it. */
1028
+ std::string reference_text;
1029
+ /** Number of flow-matching steps when supported. */
1030
+ int32_t num_steps = 5;
1031
+
1032
+ /** Model-specific extra attributes serialized to JSON internally. */
1033
+ std::unordered_map<std::string, std::string> extra;
1034
+ };
1035
+
1036
+ /** @brief Configuration for offline TTS. */
1037
+ struct OfflineTtsConfig {
1038
+ /** Model configuration. */
1039
+ OfflineTtsModelConfig model;
1040
+ /** Optional ITN rule FST archive. */
1041
+ std::string rule_fsts;
1042
+ /** Optional ITN rule FAR archive. */
1043
+ std::string rule_fars;
1044
+ /** Sentence chunking limit for generation. */
1045
+ int32_t max_num_sentences = 1;
1046
+ /** Silence scale between generated sentences. */
1047
+ float silence_scale = 0.2;
1048
+ };
1049
+
1050
+ /** @brief Generated audio returned by the C++ TTS wrapper. */
1051
+ struct GeneratedAudio {
1052
+ /** Output samples normalized to `[-1, 1]`. */
1053
+ std::vector<float> samples;
1054
+ /** Output sample rate in Hz. */
1055
+ int32_t sample_rate = 0;
1056
+ };
1057
+
1058
+ /**
1059
+ * @brief TTS progress callback.
1060
+ *
1061
+ * Return 1 to continue generating and 0 to stop early.
1062
+ */
1063
+ using OfflineTtsCallback = int32_t (*)(const float *samples,
1064
+ int32_t num_samples, float progress,
1065
+ void *arg);
1066
+
1067
+ /**
1068
+ * @brief RAII wrapper for offline TTS.
1069
+ *
1070
+ * Example using Pocket TTS:
1071
+ *
1072
+ * @code
1073
+ * OfflineTtsConfig config;
1074
+ * config.model.pocket.lm_flow =
1075
+ * "./sherpa-onnx-pocket-tts-int8-2026-01-26/lm_flow.int8.onnx";
1076
+ * config.model.pocket.lm_main =
1077
+ * "./sherpa-onnx-pocket-tts-int8-2026-01-26/lm_main.int8.onnx";
1078
+ * config.model.pocket.encoder =
1079
+ * "./sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx";
1080
+ * config.model.pocket.decoder =
1081
+ * "./sherpa-onnx-pocket-tts-int8-2026-01-26/decoder.int8.onnx";
1082
+ * config.model.pocket.text_conditioner =
1083
+ * "./sherpa-onnx-pocket-tts-int8-2026-01-26/text_conditioner.onnx";
1084
+ * config.model.pocket.vocab_json =
1085
+ * "./sherpa-onnx-pocket-tts-int8-2026-01-26/vocab.json";
1086
+ * config.model.pocket.token_scores_json =
1087
+ * "./sherpa-onnx-pocket-tts-int8-2026-01-26/token_scores.json";
1088
+ * @endcode
1089
+ */
1090
+ class SHERPA_ONNX_API OfflineTts
1091
+ : public MoveOnly<OfflineTts, SherpaOnnxOfflineTts> {
1092
+ public:
1093
+ /** @brief Create an offline TTS engine. */
1094
+ static OfflineTts Create(const OfflineTtsConfig &config);
1095
+
1096
+ /** @brief Destroy the wrapped C handle. */
1097
+ void Destroy(const SherpaOnnxOfflineTts *p) const;
1098
+
1099
+ /** @brief Return the output sample rate of generated audio. */
1100
+ int32_t SampleRate() const;
1101
+
1102
+ /** @brief Return the number of supported speakers. */
1103
+ int32_t NumSpeakers() const;
1104
+
1105
+ /**
1106
+ * @brief Generate speech using the simple speaker-id and speed interface.
1107
+ *
1108
+ * This overload mirrors the legacy/simple TTS API. Prefer the
1109
+ * `GenerationConfig` overload for new code.
1110
+ */
1111
+ GeneratedAudio Generate(const std::string &text, int32_t sid = 0,
1112
+ float speed = 1.0,
1113
+ OfflineTtsCallback callback = nullptr,
1114
+ void *arg = nullptr) const;
1115
+
1116
+ /** @brief Generate speech using the advanced generation configuration. */
1117
+ GeneratedAudio Generate(const std::string &text,
1118
+ const GenerationConfig &config,
1119
+ OfflineTtsCallback callback = nullptr,
1120
+ void *arg = nullptr) const;
1121
+
1122
+ /** @brief Like Generate(), but returns a shared pointer to the result. */
1123
+ std::shared_ptr<GeneratedAudio> Generate2(
1124
+ const std::string &text, int32_t sid = 0, float speed = 1.0,
1125
+ OfflineTtsCallback callback = nullptr, void *arg = nullptr) const;
1126
+
1127
+ /** @brief Like the advanced Generate() overload, but returns a shared
1128
+ * pointer. */
1129
+ std::shared_ptr<GeneratedAudio> Generate2(
1130
+ const std::string &text, const GenerationConfig &config,
1131
+ OfflineTtsCallback callback = nullptr, void *arg = nullptr) const;
1132
+
1133
+ private:
1134
+ explicit OfflineTts(const SherpaOnnxOfflineTts *p);
1135
+ };
1136
+
1137
+ // ============================================================
1138
+ // For Keyword Spotter
1139
+ // ============================================================
1140
+
1141
+ /** @brief Current keyword spotting result copied into C++ containers. */
1142
+ struct KeywordResult {
1143
+ /** Triggered keyword text. */
1144
+ std::string keyword;
1145
+ /** Decoded token sequence. */
1146
+ std::vector<std::string> tokens;
1147
+ /** Per-token timestamps in seconds. */
1148
+ std::vector<float> timestamps;
1149
+ /** Segment start time in seconds. */
1150
+ float start_time = 0.0f;
1151
+ /** JSON representation of the result. */
1152
+ std::string json;
1153
+ };
1154
+
1155
+ /** @brief Configuration for the C++ keyword spotting wrapper. */
1156
+ struct KeywordSpotterConfig {
1157
+ /** Feature extraction configuration. */
1158
+ FeatureConfig feat_config;
1159
+ /** Streaming acoustic model configuration. */
1160
+ OnlineModelConfig model_config;
1161
+ /** Maximum number of active paths. */
1162
+ int32_t max_active_paths = 4;
1163
+ /** Number of trailing blanks required before finalizing a trigger. */
1164
+ int32_t num_trailing_blanks = 1;
1165
+ /** Keyword score bonus. */
1166
+ float keywords_score = 1.0f;
1167
+ /** Detection threshold. */
1168
+ float keywords_threshold = 0.25f;
1169
+ /** Keyword file. */
1170
+ std::string keywords_file;
1171
+ /** In-memory keyword definitions. */
1172
+ std::string keywords_buf;
1173
+ };
1174
+
1175
+ /** @brief RAII wrapper for keyword spotting. */
1176
+ class SHERPA_ONNX_API KeywordSpotter
1177
+ : public MoveOnly<KeywordSpotter, SherpaOnnxKeywordSpotter> {
1178
+ public:
1179
+ /** @brief Create a keyword spotter from a config struct. */
1180
+ static KeywordSpotter Create(const KeywordSpotterConfig &config);
1181
+
1182
+ /** @brief Destroy the wrapped C handle. */
1183
+ void Destroy(const SherpaOnnxKeywordSpotter *p) const;
1184
+
1185
+ /** @brief Create a keyword stream using configured keywords. */
1186
+ OnlineStream CreateStream() const;
1187
+
1188
+ /** @brief Create a keyword stream with inline extra or replacement keywords.
1189
+ */
1190
+ OnlineStream CreateStream(const std::string &keywords) const;
1191
+
1192
+ /** @brief Check whether the stream has enough data to decode. */
1193
+ bool IsReady(const OnlineStream *s) const;
1194
+
1195
+ /** @brief Decode one ready stream. */
1196
+ void Decode(const OnlineStream *s) const;
1197
+
1198
+ /** @brief Decode multiple ready streams in parallel. */
1199
+ void Decode(const OnlineStream *ss, int32_t n) const;
1200
+
1201
+ /** @brief Reset a stream after a keyword trigger. */
1202
+ void Reset(const OnlineStream *s) const;
1203
+
1204
+ /** @brief Return the copied keyword spotting result for a stream. */
1205
+ KeywordResult GetResult(const OnlineStream *s) const;
1206
+
1207
+ private:
1208
+ explicit KeywordSpotter(const SherpaOnnxKeywordSpotter *p);
1209
+ };
1210
+
1211
+ /** @brief GTCRN speech denoiser model configuration. */
1212
+ struct OfflineSpeechDenoiserGtcrnModelConfig {
1213
+ /** Model ONNX file. */
1214
+ std::string model;
1215
+ };
1216
+
1217
+ /** @brief DPDFNet speech denoiser model configuration. */
1218
+ struct OfflineSpeechDenoiserDpdfNetModelConfig {
1219
+ /** Model ONNX file. */
1220
+ std::string model;
1221
+ };
1222
+
1223
+ /**
1224
+ * @brief Speech denoiser model configuration.
1225
+ *
1226
+ * Configure exactly one model family. If multiple model families are set, one
1227
+ * is chosen and the choice is implementation-defined.
1228
+ */
1229
+ struct OfflineSpeechDenoiserModelConfig {
1230
+ /** GTCRN configuration. */
1231
+ OfflineSpeechDenoiserGtcrnModelConfig gtcrn;
1232
+ /** DPDFNet configuration. */
1233
+ OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet;
1234
+ /** Number of inference threads. */
1235
+ int32_t num_threads = 1;
1236
+ /** Enable verbose debug logging. */
1237
+ bool debug = false;
1238
+ /** Execution provider such as `"cpu"`. */
1239
+ std::string provider = "cpu";
1240
+ };
1241
+
1242
+ /** @brief Configuration for offline speech denoising. */
1243
+ struct OfflineSpeechDenoiserConfig {
1244
+ /** Model configuration. */
1245
+ OfflineSpeechDenoiserModelConfig model;
1246
+ };
1247
+
1248
+ /** @brief Denoised waveform returned by speech enhancement wrappers. */
1249
+ struct DenoisedAudio {
1250
+ /** Output samples normalized to `[-1, 1]`. */
1251
+ std::vector<float> samples;
1252
+ /** Output sample rate in Hz. */
1253
+ int32_t sample_rate = 0;
1254
+ };
1255
+
1256
+ /** @brief RAII wrapper for offline speech denoising. */
1257
+ class SHERPA_ONNX_API OfflineSpeechDenoiser
1258
+ : public MoveOnly<OfflineSpeechDenoiser, SherpaOnnxOfflineSpeechDenoiser> {
1259
+ public:
1260
+ /** @brief Create an offline speech denoiser. */
1261
+ static OfflineSpeechDenoiser Create(
1262
+ const OfflineSpeechDenoiserConfig &config);
1263
+
1264
+ /** @brief Destroy the wrapped C handle. */
1265
+ void Destroy(const SherpaOnnxOfflineSpeechDenoiser *p) const;
1266
+
1267
+ /** @brief Run denoising on a complete waveform. */
1268
+ DenoisedAudio Run(const float *samples, int32_t n, int32_t sample_rate) const;
1269
+
1270
+ /** @brief Return the expected input sample rate. */
1271
+ int32_t GetSampleRate() const;
1272
+
1273
+ private:
1274
+ explicit OfflineSpeechDenoiser(const SherpaOnnxOfflineSpeechDenoiser *p);
1275
+ };
1276
+
1277
+ /** @brief Configuration for online speech denoising. */
1278
+ struct OnlineSpeechDenoiserConfig {
1279
+ /** Model configuration. */
1280
+ OfflineSpeechDenoiserModelConfig model;
1281
+ };
1282
+
1283
+ /** @brief RAII wrapper for online speech denoising. */
1284
+ class SHERPA_ONNX_API OnlineSpeechDenoiser
1285
+ : public MoveOnly<OnlineSpeechDenoiser, SherpaOnnxOnlineSpeechDenoiser> {
1286
+ public:
1287
+ /** @brief Create an online speech denoiser. */
1288
+ static OnlineSpeechDenoiser Create(const OnlineSpeechDenoiserConfig &config);
1289
+
1290
+ /** @brief Destroy the wrapped C handle. */
1291
+ void Destroy(const SherpaOnnxOnlineSpeechDenoiser *p) const;
1292
+
1293
+ /** @brief Process one chunk of streaming audio. */
1294
+ DenoisedAudio Run(const float *samples, int32_t n, int32_t sample_rate) const;
1295
+
1296
+ /** @brief Flush buffered audio and reset the denoiser. */
1297
+ DenoisedAudio Flush() const;
1298
+
1299
+ /** @brief Reset the denoiser for a new stream. */
1300
+ void Reset() const;
1301
+
1302
+ /** @brief Return the expected input sample rate. */
1303
+ int32_t GetSampleRate() const;
1304
+
1305
+ /** @brief Return the recommended frame shift in samples for streaming input.
1306
+ */
1307
+ int32_t GetFrameShiftInSamples() const;
1308
+
1309
+ private:
1310
+ explicit OnlineSpeechDenoiser(const SherpaOnnxOnlineSpeechDenoiser *p);
1311
+ };
1312
+
1313
+ // ==============================
1314
+ // VAD
1315
+ // ==============================
1316
+
1317
+ /** @brief Silero VAD model configuration. */
1318
+ struct SileroVadModelConfig {
1319
+ /** Model ONNX file. */
1320
+ std::string model;
1321
+ /** Detection threshold. */
1322
+ float threshold = 0.5;
1323
+ /** Minimum silence duration in seconds. */
1324
+ float min_silence_duration = 0.5;
1325
+ /** Minimum speech duration in seconds. */
1326
+ float min_speech_duration = 0.25;
1327
+ /** Window size in samples. */
1328
+ int32_t window_size = 512;
1329
+ /** Maximum speech duration in seconds before forced split. */
1330
+ float max_speech_duration = 20;
1331
+ };
1332
+
1333
+ /** @brief Ten VAD model configuration. */
1334
+ struct TenVadModelConfig {
1335
+ /** Model ONNX file. */
1336
+ std::string model;
1337
+ /** Detection threshold. */
1338
+ float threshold = 0.5;
1339
+ /** Minimum silence duration in seconds. */
1340
+ float min_silence_duration = 0.5;
1341
+ /** Minimum speech duration in seconds. */
1342
+ float min_speech_duration = 0.25;
1343
+ /** Window size in samples. */
1344
+ int32_t window_size = 256;
1345
+ /** Maximum speech duration in seconds before forced split. */
1346
+ float max_speech_duration = 20;
1347
+ };
1348
+
1349
+ /**
1350
+ * @brief VAD model configuration.
1351
+ *
1352
+ * Configure exactly one model family. If multiple model families are set, one
1353
+ * is chosen and the choice is implementation-defined.
1354
+ */
1355
+ struct VadModelConfig {
1356
+ /** Silero VAD configuration. */
1357
+ SileroVadModelConfig silero_vad;
1358
+ /** Ten VAD configuration. */
1359
+ TenVadModelConfig ten_vad;
1360
+
1361
+ /** Input sample rate in Hz. */
1362
+ int32_t sample_rate = 16000;
1363
+ /** Number of inference threads. */
1364
+ int32_t num_threads = 1;
1365
+ /** Execution provider such as `"cpu"`. */
1366
+ std::string provider = "cpu";
1367
+ /** Enable verbose debug logging. */
1368
+ bool debug = false;
1369
+ };
1370
+
1371
+ /** @brief One speech segment produced by the VAD wrapper. */
1372
+ struct SpeechSegment {
1373
+ /** Start sample index relative to the processed audio timeline. */
1374
+ int32_t start = 0;
1375
+ /** Speech samples for the segment. */
1376
+ std::vector<float> samples;
1377
+ };
1378
+
1379
+ /** @brief RAII wrapper for the circular buffer helper used by VAD. */
1380
+ class SHERPA_ONNX_API CircularBuffer
1381
+ : public MoveOnly<CircularBuffer, SherpaOnnxCircularBuffer> {
1382
+ public:
1383
+ /** @brief Create a circular buffer with the given capacity in samples. */
1384
+ static CircularBuffer Create(int32_t capacity);
1385
+
1386
+ /** @brief Destroy the wrapped C handle. */
1387
+ void Destroy(const SherpaOnnxCircularBuffer *p) const;
1388
+
1389
+ /** @brief Append samples to the buffer. */
1390
+ void Push(const float *p, int32_t n) const;
1391
+
1392
+ /** @brief Copy a contiguous span from the buffer. */
1393
+ std::vector<float> Get(int32_t start_index, int32_t n) const;
1394
+
1395
+ /** @brief Remove samples from the head of the buffer. */
1396
+ void Pop(int32_t n) const;
1397
+
1398
+ /** @brief Return the number of stored samples. */
1399
+ int32_t Size() const;
1400
+
1401
+ /** @brief Return the current head index. */
1402
+ int32_t Head() const;
1403
+
1404
+ /** @brief Reset the buffer to empty. */
1405
+ void Reset() const;
1406
+
1407
+ private:
1408
+ explicit CircularBuffer(const SherpaOnnxCircularBuffer *p);
1409
+ };
1410
+
1411
+ /**
1412
+ * @brief RAII wrapper for voice activity detection.
1413
+ *
1414
+ * The wrapper collects detected speech segments internally. Use `IsEmpty()`,
1415
+ * `Front()`, and `Pop()` to consume them.
1416
+ */
1417
+ class SHERPA_ONNX_API VoiceActivityDetector
1418
+ : public MoveOnly<VoiceActivityDetector, SherpaOnnxVoiceActivityDetector> {
1419
+ public:
1420
+ /** @brief Create a VAD instance. */
1421
+ static VoiceActivityDetector Create(const VadModelConfig &config,
1422
+ float buffer_size_in_seconds);
1423
+
1424
+ /** @brief Destroy the wrapped C handle. */
1425
+ void Destroy(const SherpaOnnxVoiceActivityDetector *p) const;
1426
+
1427
+ /** @brief Feed more audio samples to the detector. */
1428
+ void AcceptWaveform(const float *samples, int32_t n) const;
1429
+
1430
+ /** @brief Check whether no speech segments are currently queued. */
1431
+ bool IsEmpty() const;
1432
+
1433
+ /** @brief Check whether speech is currently detected. */
1434
+ bool IsDetected() const;
1435
+
1436
+ /** @brief Remove the front queued speech segment. */
1437
+ void Pop() const;
1438
+
1439
+ /** @brief Remove all queued speech segments. */
1440
+ void Clear() const;
1441
+
1442
+ /** @brief Return the front queued speech segment. */
1443
+ SpeechSegment Front() const;
1444
+
1445
+ /** @brief Like Front(), but returns the segment in a shared pointer. */
1446
+ std::shared_ptr<SpeechSegment> FrontPtr() const;
1447
+
1448
+ /** @brief Reset the detector state. */
1449
+ void Reset() const;
1450
+
1451
+ /** @brief Flush buffered context at end of input. */
1452
+ void Flush() const;
1453
+
1454
+ private:
1455
+ explicit VoiceActivityDetector(const SherpaOnnxVoiceActivityDetector *p);
1456
+ };
1457
+
1458
+ /** @brief RAII wrapper for linear resampling. */
1459
+ class SHERPA_ONNX_API LinearResampler
1460
+ : public MoveOnly<LinearResampler, SherpaOnnxLinearResampler> {
1461
+ public:
1462
+ /** @brief Construct an empty wrapper. */
1463
+ LinearResampler() = default;
1464
+ /** @brief Create a linear resampler. */
1465
+ static LinearResampler Create(int32_t samp_rate_in_hz,
1466
+ int32_t samp_rate_out_hz,
1467
+ float filter_cutoff_hz, int32_t num_zeros);
1468
+
1469
+ /** @brief Destroy the wrapped C handle. */
1470
+ void Destroy(const SherpaOnnxLinearResampler *p) const;
1471
+
1472
+ /** @brief Reset the resampler state. */
1473
+ void Reset() const;
1474
+
1475
+ /** @brief Resample one chunk of input audio. */
1476
+ std::vector<float> Resample(const float *input, int32_t input_dim,
1477
+ bool flush) const;
1478
+
1479
+ /** @brief Return the input sample rate in Hz. */
1480
+ int32_t GetInputSamplingRate() const;
1481
+ /** @brief Return the output sample rate in Hz. */
1482
+ int32_t GetOutputSamplingRate() const;
1483
+
1484
+ private:
1485
+ explicit LinearResampler(const SherpaOnnxLinearResampler *p);
1486
+ };
1487
+
1488
+ /** @brief Return the sherpa-onnx version string as a C++ string. */
1489
+ SHERPA_ONNX_API std::string GetVersionStr();
1490
+ /** @brief Return the build Git SHA1 as a C++ string. */
1491
+ SHERPA_ONNX_API std::string GetGitSha1();
1492
+ /** @brief Return the build Git date as a C++ string. */
1493
+ SHERPA_ONNX_API std::string GetGitDate();
1494
+ /** @brief Return `true` if a file exists. */
1495
+ SHERPA_ONNX_API bool FileExists(const std::string &filename);
1496
+
1497
+ // ============================================================================
1498
+ // Offline Punctuation
1499
+ // ============================================================================
1500
+ /** @brief Offline punctuation model configuration. */
1501
+ struct OfflinePunctuationModelConfig {
1502
+ /** Model file. */
1503
+ std::string ct_transformer;
1504
+ /** Number of inference threads. */
1505
+ int32_t num_threads = 1;
1506
+ /** Enable verbose debug logging. */
1507
+ bool debug = false;
1508
+ /** Execution provider such as `"cpu"`. */
1509
+ std::string provider = "cpu";
1510
+ };
1511
+
1512
+ /** @brief Configuration for offline punctuation. */
1513
+ struct OfflinePunctuationConfig {
1514
+ /** Model configuration. */
1515
+ OfflinePunctuationModelConfig model;
1516
+ };
1517
+
1518
+ /** @brief RAII wrapper for offline punctuation restoration. */
1519
+ class SHERPA_ONNX_API OfflinePunctuation
1520
+ : public MoveOnly<OfflinePunctuation, SherpaOnnxOfflinePunctuation> {
1521
+ public:
1522
+ /** @brief Create an offline punctuation model. */
1523
+ static OfflinePunctuation Create(const OfflinePunctuationConfig &config);
1524
+
1525
+ /** @brief Destroy the wrapped C handle. */
1526
+ void Destroy(const SherpaOnnxOfflinePunctuation *p) const;
1527
+
1528
+ /** @brief Add punctuation to a complete input text. */
1529
+ std::string AddPunctuation(const std::string &text) const;
1530
+
1531
+ private:
1532
+ explicit OfflinePunctuation(const SherpaOnnxOfflinePunctuation *p);
1533
+ };
1534
+
1535
+ // ============================================================================
1536
+ // Online Punctuation
1537
+ // ============================================================================
1538
+ /** @brief Online punctuation model configuration. */
1539
+ struct OnlinePunctuationModelConfig {
1540
+ /** Model file. */
1541
+ std::string cnn_bilstm;
1542
+ /** BPE vocabulary file. */
1543
+ std::string bpe_vocab;
1544
+ /** Number of inference threads. */
1545
+ int32_t num_threads = 1;
1546
+ /** Enable verbose debug logging. */
1547
+ bool debug = false;
1548
+ /** Execution provider such as `"cpu"`. */
1549
+ std::string provider = "cpu";
1550
+ };
1551
+
1552
+ /** @brief Configuration for online punctuation. */
1553
+ struct OnlinePunctuationConfig {
1554
+ /** Model configuration. */
1555
+ OnlinePunctuationModelConfig model;
1556
+ };
1557
+
1558
+ /** @brief RAII wrapper for online punctuation restoration. */
1559
+ class SHERPA_ONNX_API OnlinePunctuation
1560
+ : public MoveOnly<OnlinePunctuation, SherpaOnnxOnlinePunctuation> {
1561
+ public:
1562
+ /** @brief Create an online punctuation model. */
1563
+ static OnlinePunctuation Create(const OnlinePunctuationConfig &config);
1564
+
1565
+ /** @brief Destroy the wrapped C handle. */
1566
+ void Destroy(const SherpaOnnxOnlinePunctuation *p) const;
1567
+
1568
+ /** @brief Add punctuation to one input text chunk. */
1569
+ std::string AddPunctuation(const std::string &text) const;
1570
+
1571
+ private:
1572
+ explicit OnlinePunctuation(const SherpaOnnxOnlinePunctuation *p);
1573
+ };
1574
+
1575
+ // ============================================================================
1576
+ // Audio tagging
1577
+ // ============================================================================
1578
+ /** @brief Zipformer audio-tagging model configuration. */
1579
+ struct OfflineZipformerAudioTaggingModelConfig {
1580
+ /** Model file. */
1581
+ std::string model;
1582
+ };
1583
+
1584
+ /**
1585
+ * @brief Audio-tagging model configuration.
1586
+ *
1587
+ * Configure exactly one model family. If multiple model families are set, one
1588
+ * is chosen and the choice is implementation-defined.
1589
+ */
1590
+ struct AudioTaggingModelConfig {
1591
+ /** Zipformer model configuration. */
1592
+ OfflineZipformerAudioTaggingModelConfig zipformer;
1593
+ /** Alternative CED model file. */
1594
+ std::string ced;
1595
+ /** Number of inference threads. */
1596
+ int32_t num_threads = 1;
1597
+ /** Enable verbose debug logging. */
1598
+ bool debug = false;
1599
+ /** Execution provider such as `"cpu"`. */
1600
+ std::string provider = "cpu";
1601
+ };
1602
+
1603
+ /** @brief Configuration for audio tagging. */
1604
+ struct AudioTaggingConfig {
1605
+ /** Model configuration. */
1606
+ AudioTaggingModelConfig model;
1607
+ /** CSV file containing label names. */
1608
+ std::string labels;
1609
+ /** Default number of results to return. */
1610
+ int32_t top_k = 5;
1611
+ };
1612
+
1613
+ /** @brief One audio-tagging event returned by the C++ wrapper. */
1614
+ struct AudioEvent {
1615
+ /** Event label. */
1616
+ std::string name;
1617
+ /** Class index. */
1618
+ int32_t index;
1619
+ /** Probability or confidence score. */
1620
+ float prob;
1621
+ };
1622
+
1623
+ /** @brief RAII wrapper for audio tagging. */
1624
+ class SHERPA_ONNX_API AudioTagging
1625
+ : public MoveOnly<AudioTagging, SherpaOnnxAudioTagging> {
1626
+ public:
1627
+ /** @brief Create an audio tagger. */
1628
+ static AudioTagging Create(const AudioTaggingConfig &config);
1629
+
1630
+ /** @brief Destroy the wrapped C handle. */
1631
+ void Destroy(const SherpaOnnxAudioTagging *p) const;
1632
+
1633
+ /** @brief Create an offline stream for tagging. */
1634
+ OfflineStream CreateStream() const;
1635
+ /**
1636
+ * @brief Run audio tagging and return copied results.
1637
+ *
1638
+ * When `top_k == -1`, the wrapper uses `config.top_k`. When `top_k > 0`,
1639
+ * that argument overrides the configured default.
1640
+ */
1641
+ std::vector<AudioEvent> Compute(const OfflineStream *s, int32_t top_k = -1);
1642
+
1643
+ /** @brief Like Compute(), but returns the result vector in a shared pointer.
1644
+ */
1645
+ std::shared_ptr<std::vector<AudioEvent>> ComputePtr(const OfflineStream *s,
1646
+ int32_t top_k = -1);
1647
+
1648
+ private:
1649
+ explicit AudioTagging(const SherpaOnnxAudioTagging *p);
1650
+ };
1651
+
1652
+ // ==============================
1653
+ // Source Separation
1654
+ // ==============================
1655
+
1656
+ /** @brief Spleeter source-separation model configuration. */
1657
+ struct OfflineSourceSeparationSpleeterModelConfig {
1658
+ /** Path to the vocals ONNX model. */
1659
+ std::string vocals;
1660
+ /** Path to the accompaniment ONNX model. */
1661
+ std::string accompaniment;
1662
+ };
1663
+
1664
+ /** @brief UVR (MDX-Net) source-separation model configuration. */
1665
+ struct OfflineSourceSeparationUvrModelConfig {
1666
+ /** Path to the UVR ONNX model. */
1667
+ std::string model;
1668
+ };
1669
+
1670
+ /**
1671
+ * @brief Source-separation model configuration.
1672
+ *
1673
+ * Configure exactly one model family (Spleeter or UVR).
1674
+ */
1675
+ struct OfflineSourceSeparationModelConfig {
1676
+ /** Spleeter configuration. */
1677
+ OfflineSourceSeparationSpleeterModelConfig spleeter;
1678
+ /** UVR configuration. */
1679
+ OfflineSourceSeparationUvrModelConfig uvr;
1680
+ /** Number of inference threads. */
1681
+ int32_t num_threads = 1;
1682
+ /** Enable verbose debug logging. */
1683
+ bool debug = false;
1684
+ /** Execution provider such as `"cpu"`. */
1685
+ std::string provider = "cpu";
1686
+ };
1687
+
1688
+ /** @brief Configuration for offline source separation. */
1689
+ struct OfflineSourceSeparationConfig {
1690
+ /** Model configuration. */
1691
+ OfflineSourceSeparationModelConfig model;
1692
+ };
1693
+
1694
+ /** @brief A single stem (output track) with one or more channels. */
1695
+ struct SourceSeparationStem {
1696
+ /** samples[c] contains the sample array for channel c. */
1697
+ std::vector<std::vector<float>> samples;
1698
+ };
1699
+
1700
+ /** @brief Output of a source-separation run. */
1701
+ struct SourceSeparationOutput {
1702
+ /** Separated stems. */
1703
+ std::vector<SourceSeparationStem> stems;
1704
+ /** Sample rate in Hz. */
1705
+ int32_t sample_rate = 0;
1706
+ };
1707
+
1708
+ /** @brief RAII wrapper for offline source separation. */
1709
+ class SHERPA_ONNX_API OfflineSourceSeparation
1710
+ : public MoveOnly<OfflineSourceSeparation,
1711
+ SherpaOnnxOfflineSourceSeparation> {
1712
+ public:
1713
+ /** @brief Create an offline source separation engine. */
1714
+ static OfflineSourceSeparation Create(
1715
+ const OfflineSourceSeparationConfig &config);
1716
+
1717
+ /** @brief Destroy the wrapped C handle. */
1718
+ void Destroy(const SherpaOnnxOfflineSourceSeparation *p) const;
1719
+
1720
+ /**
1721
+ * @brief Run source separation on multi-channel audio.
1722
+ *
1723
+ * @param samples samples[c] is a float array for channel c.
1724
+ * @param num_channels Number of input channels.
1725
+ * @param num_samples Number of samples per channel.
1726
+ * @param sample_rate Input sample rate in Hz.
1727
+ * @return Separated stems, or an empty output on error.
1728
+ */
1729
+ SourceSeparationOutput Process(const float *const *samples,
1730
+ int32_t num_channels, int32_t num_samples,
1731
+ int32_t sample_rate) const;
1732
+
1733
+ /** @brief Return the output sample rate. */
1734
+ int32_t GetOutputSampleRate() const;
1735
+
1736
+ /** @brief Return the number of stems produced. */
1737
+ int32_t GetNumberOfStems() const;
1738
+
1739
+ private:
1740
+ explicit OfflineSourceSeparation(const SherpaOnnxOfflineSourceSeparation *p);
1741
+ };
1742
+
1743
+ } // namespace sherpa_onnx::cxx
1744
+
1745
+ #endif // SHERPA_ONNX_C_API_CXX_API_H_