echogarden 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/api/Synthesis.d.ts.map +1 -1
- package/dist/api/Synthesis.js +31 -21
- package/dist/api/Synthesis.js.map +1 -1
- package/dist/nlp/TextNormalizer.js +1 -1
- package/dist/nlp/TextNormalizer.js.map +1 -1
- package/dist/synthesis/EspeakTTS.d.ts.map +1 -1
- package/dist/synthesis/EspeakTTS.js +20 -5
- package/dist/synthesis/EspeakTTS.js.map +1 -1
- package/dist/utilities/Timer.d.ts +7 -3
- package/dist/utilities/Timer.d.ts.map +1 -1
- package/dist/utilities/Timer.js +42 -30
- package/dist/utilities/Timer.js.map +1 -1
- package/docs/API.md +51 -34
- package/docs/CLI.md +41 -6
- package/docs/Contributing.md +1 -1
- package/docs/Development.md +22 -18
- package/docs/Engines.md +0 -1
- package/docs/Licenses.md +2 -0
- package/docs/Options.md +56 -4
- package/docs/Releases.md +45 -14
- package/docs/Server.md +14 -5
- package/docs/Tasklist.md +37 -1
- package/docs/Technical.md +1 -0
- package/package.json +16 -14
- package/src/api/Synthesis.ts +39 -27
- package/src/nlp/TextNormalizer.ts +1 -2
- package/src/synthesis/EspeakTTS.ts +21 -6
- package/src/utilities/Timer.ts +57 -37
- package/tsconfig.json +51 -53
- package/dist/encodings/HtmlEscape.d.ts +0 -2
- package/dist/encodings/HtmlEscape.d.ts.map +0 -1
- package/dist/encodings/HtmlEscape.js +0 -30
- package/dist/encodings/HtmlEscape.js.map +0 -1
- package/dist/encodings/LEB128.d.ts +0 -2
- package/dist/encodings/LEB128.d.ts.map +0 -1
- package/dist/encodings/LEB128.js +0 -2
- package/dist/encodings/LEB128.js.map +0 -1
- package/dist/utilities/StringBuilder.d.ts +0 -11
- package/dist/utilities/StringBuilder.d.ts.map +0 -1
- package/dist/utilities/StringBuilder.js +0 -39
- package/dist/utilities/StringBuilder.js.map +0 -1
package/docs/Releases.md
CHANGED
|
@@ -7,6 +7,7 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
|
|
|
7
7
|
## `1.0.0` (April 12, 2024)
|
|
8
8
|
|
|
9
9
|
**New features**:
|
|
10
|
+
|
|
10
11
|
* Add support for [`whisper.cpp`](https://github.com/ggerganov/whisper.cpp), a C++ port of OpenAI's Whisper speech recognition architecture. It is faster than the integrated `whisper` engine, supports large models, and GPU processing. It can now be used for recognition, speech translation and alignment (via `dtw-ra`), though its word timestamps are less accurate than the integrated `whisper` engine.
|
|
11
12
|
* Add the [MDX-NET](https://github.com/kuielab/mdx-net/) source separation model, enabling vocal tracks to be extracted from music and speech audio using the new `isolate` operation
|
|
12
13
|
* Add support for OpenAI cloud platform's [speech recognition and translation services](https://platform.openai.com/docs/guides/speech-to-text)
|
|
@@ -18,17 +19,19 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
|
|
|
18
19
|
* Expose more options for the Elevenlabs engine
|
|
19
20
|
|
|
20
21
|
**Enhancements**:
|
|
22
|
+
|
|
21
23
|
* `detect-voice-activity` now also outputs the cropped voice when an audio file is given as output, with `.cropped` suffix added
|
|
22
24
|
* Improved word segmentation of whisper engine. Words spanning multiple recognized time segments are now split. Word timestamp ranges don't overlap non-speech sections.
|
|
23
25
|
* Whisper and Whisper.cpp outputs now include timestamps for individual recognized tokens
|
|
24
26
|
|
|
25
27
|
**Behavioral and breaking changes**:
|
|
28
|
+
|
|
26
29
|
* Minimal required node version changed to `18.0.0`
|
|
27
30
|
* All recognition, alignment, translation and language recognition operations first apply the new adaptive gate VAD (can be changed to any other VAD engine via the `vad.` option prefix) and remove any sections that are not identified as containing voice, before starting processing. This should improve results in most cases, and reduce processing time
|
|
28
31
|
* To reduce Whisper hallucinations and repetition loops, these changes were made:
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
+
* Pre-cropping by default can significantly help with reducing hallucinations, giving the model less "empty space" to hallucinate on
|
|
33
|
+
* Enable new token-level repetition suppression (`suppressRepetition = true`) during decoding
|
|
34
|
+
* Disable decoding of timestamp tokens by default (`decodeTimestampTokens = false`), since more accurate timing is already extracted via cross-attention weight alignment. For unclear reasons, this can significantly reduce the occurrence of token repetition loops, and increases word timestamp accuracy. However, there are cases where this causes the model to end a part prematurely, especially in singing and less speech-like voice segments. In those cases the it can be enabled with `decodeTimestampTokens = true`
|
|
32
35
|
* When `transcribe`, `align` or `translate-speech` operations are run with `--isolate` enabled, they will output the isolated part in `some-output-file.isolated.wav` and background part (isolated subtracted from original) in `some-output-file.background.wav` (any supported codec other than `wav` can be used - this is just an example)
|
|
33
36
|
* VAD operations now return a timeline including only the active sections, labeled as `active`
|
|
34
37
|
* When the specified language is not English, but an `.en` Whisper model was specified, a warning would be shown, and the model will be automatically switched to the corresponding multilingual model (omitting the `.en`), instead of producing an error
|
|
@@ -40,6 +43,7 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
|
|
|
40
43
|
* `dryMixGainDb` denoising option renamed to `dryMixGain`
|
|
41
44
|
|
|
42
45
|
**Fixes**:
|
|
46
|
+
|
|
43
47
|
* Integrated `whisper` engine now uses `tiktoken` to tokenize text, which produces near identical tokenization compared to the official Python implementation. This fixes issues with decoding Chinese characters, correctly encoding prompts, and several other languages with characters that span multiple tokens
|
|
44
48
|
* Fix voice language lists for multilingual voices in Elevenlabs TTS engine
|
|
45
49
|
* Fix runtime error with `rnnoise` when used as VAD engine
|
|
@@ -53,6 +57,7 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
|
|
|
53
57
|
* Various other fixes
|
|
54
58
|
|
|
55
59
|
**Documentation**:
|
|
60
|
+
|
|
56
61
|
* Add new [release notes](./Releases.md) page. Releases before `1.0.x` were retroactively documented based on commit history (may not exactly detail all historical changes)
|
|
57
62
|
* Add new [development page](./Development.md) with a quick guide for setting up a development environment to make changes to the Echogarden codebase
|
|
58
63
|
* Options reference page reorganized and updated with some missing information
|
|
@@ -60,24 +65,27 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
|
|
|
60
65
|
## `0.12.x` (March 16, 2024)
|
|
61
66
|
|
|
62
67
|
**New features**:
|
|
68
|
+
|
|
63
69
|
* Add global options support to API
|
|
64
70
|
* Add updated `ffmpeg` internal packages for many platforms, including pre-signed binaries for macOS
|
|
65
71
|
|
|
66
72
|
**Enhancements**:
|
|
67
73
|
|
|
68
74
|
**Fixes**:
|
|
75
|
+
|
|
69
76
|
* Fix issue with Polish TTS in eSpeak NG
|
|
70
77
|
* Fix warning about `punycode` module in the CLI
|
|
71
78
|
|
|
72
79
|
**Other**:
|
|
73
|
-
* Include `package-lock.json` in repository
|
|
74
80
|
|
|
81
|
+
* Include `package-lock.json` in repository
|
|
75
82
|
|
|
76
83
|
## `0.11.x` (August 17, 2023)
|
|
77
84
|
|
|
78
85
|
Many features, enhancements, and fixes were incrementally added over the span of 7 months, up to March 2024.
|
|
79
86
|
|
|
80
87
|
**New features**:
|
|
88
|
+
|
|
81
89
|
* Partial rewrite of subtitle generation methods. Adds many features and options
|
|
82
90
|
* Expose subtitle methods to API
|
|
83
91
|
* Add support for multi-pass (AKA hierarchical) DTW alignment
|
|
@@ -85,72 +93,82 @@ Many features, enhancements, and fixes were incrementally added over the span of
|
|
|
85
93
|
* Add support for Klatt synthesis in `espeak` engine
|
|
86
94
|
|
|
87
95
|
**Enhancements**:
|
|
96
|
+
|
|
88
97
|
* Adds a total of 14 new VITS voices
|
|
89
98
|
* Retry on failure for `microsoft-edge` and `google-translate` TTS engines
|
|
90
99
|
|
|
91
100
|
**Fixes**:
|
|
101
|
+
|
|
92
102
|
* Don't error when empty audio is returned in `microsoft-edge` response
|
|
93
103
|
* Fix audio playback in macOS
|
|
94
104
|
* Many other fixes
|
|
95
105
|
|
|
96
|
-
|
|
97
106
|
## `0.10.x` (August 2, 2023)
|
|
98
107
|
|
|
99
108
|
**New features**:
|
|
109
|
+
|
|
100
110
|
* Add developer and server APIs, allowing the package to be used as a library or server
|
|
101
111
|
* Add granularity options to DTW
|
|
102
112
|
* Add Linux SoX package
|
|
103
113
|
|
|
104
114
|
**Enhancements**:
|
|
115
|
+
|
|
105
116
|
* Adds 7 new VITS voices
|
|
106
117
|
|
|
107
118
|
**Fixes**:
|
|
108
|
-
* Many fixes
|
|
109
119
|
|
|
120
|
+
* Many fixes
|
|
110
121
|
|
|
111
122
|
## `0.9.x` (July 29, 2023)
|
|
112
123
|
|
|
113
124
|
**New features**:
|
|
125
|
+
|
|
114
126
|
* Add text offsets to timeline
|
|
115
127
|
* Include segments and sentences in recognition and speech translation timelines
|
|
116
128
|
* Improve voice information in Elevenlabs engine
|
|
117
129
|
|
|
118
130
|
**Behavioral changes**:
|
|
131
|
+
|
|
119
132
|
* Set `en_GB-alan-low` as default `en-GB` VITS voice, since `danny` was trained to accept `en-US` pronunciations, which may confuse some people
|
|
120
133
|
|
|
121
134
|
**Fixes**:
|
|
135
|
+
|
|
122
136
|
* Work around several eSpeak bugs
|
|
123
137
|
* Many fixes
|
|
124
138
|
|
|
125
|
-
|
|
126
139
|
## `0.8.x` (July 25, 2023)
|
|
127
140
|
|
|
128
141
|
**New features**:
|
|
142
|
+
|
|
129
143
|
* Add confidence to some speech recognition timelines
|
|
130
144
|
* Add `plaintext` options
|
|
131
145
|
* Add punctuation thresholds to Whisper decoder
|
|
132
146
|
* Expose subtitle configuration options to CLI
|
|
133
147
|
|
|
134
148
|
**Enhancements**:
|
|
149
|
+
|
|
135
150
|
* Improve speech language detection to work with arbitrary length audio. Split audio into overlapping parts and detect each part individually. Then average the results.
|
|
136
151
|
* Add more heteronyms
|
|
137
152
|
|
|
138
153
|
**Behavioral changes**:
|
|
154
|
+
|
|
139
155
|
* Rename `subtitles.minWords` to `subtitles.minWordsInLine`
|
|
140
156
|
|
|
141
157
|
**Fixes**:
|
|
158
|
+
|
|
142
159
|
* Improve numerical stability of softmax function
|
|
143
160
|
* Work around eSpeak bug with markers in long inputs
|
|
144
161
|
* Various fixes
|
|
145
162
|
|
|
146
|
-
|
|
147
163
|
## `0.7.x` (July 23, 2023)
|
|
148
164
|
|
|
149
165
|
**New features**:
|
|
166
|
+
|
|
150
167
|
* Alignment: Add option to accept custom lexicons
|
|
151
168
|
* Implement and add decoder temperature option to Whisper model
|
|
152
169
|
|
|
153
170
|
**Enhancements**:
|
|
171
|
+
|
|
154
172
|
* Alignment: use preprocessing and lexicons
|
|
155
173
|
* Add language detection to speech translation
|
|
156
174
|
* Add language detection options to synthesis, and set empty default options for detection in several APIs
|
|
@@ -162,13 +180,14 @@ Many features, enhancements, and fixes were incrementally added over the span of
|
|
|
162
180
|
**Behavioral changes**:
|
|
163
181
|
|
|
164
182
|
**Fixes**:
|
|
183
|
+
|
|
165
184
|
* Fix support for SSML input in eSpeak engine
|
|
166
185
|
* Various fixes
|
|
167
186
|
|
|
168
|
-
|
|
169
187
|
## `0.6.x` (July 20, 2023)
|
|
170
188
|
|
|
171
189
|
**New features**:
|
|
190
|
+
|
|
172
191
|
* Add support for custom lexicons. Change lexicon object structure to include language code and allow for multiple languages in a single lexicon
|
|
173
192
|
* Add support for SSML inputs (currently only supported by Google, Microsoft and Amazon cloud engines). Ensure they are not split to segments or sentences
|
|
174
193
|
* CLI: Add flag to enable or disable file overwriting in CLI
|
|
@@ -177,44 +196,50 @@ Many features, enhancements, and fixes were incrementally added over the span of
|
|
|
177
196
|
* Change whitespace option to include option for collapsing all whitespace
|
|
178
197
|
|
|
179
198
|
**Enhancements**:
|
|
199
|
+
|
|
180
200
|
* Change whitespace option to include option for collapsing all whitespace
|
|
181
201
|
* Add awareness of guillemets
|
|
182
202
|
* Log full language of selected voice
|
|
183
203
|
|
|
184
204
|
**Behavioral changes**:
|
|
205
|
+
|
|
185
206
|
* Set Elevenlabs defaults to mid-values
|
|
186
207
|
* Split plain text to paragraphs using double line breaks by default.
|
|
187
208
|
* Rename `awsPolly` options to `amazonPolly` to be more consistent with documentation
|
|
188
209
|
|
|
189
210
|
**Fixes**:
|
|
211
|
+
|
|
190
212
|
* Fix and update Elevenlabs engine
|
|
191
213
|
* Fix incorrect gender properties for some VITS voices
|
|
192
214
|
* Convert to plaintext before detecting language when input is SSML.
|
|
193
215
|
* Various fixes
|
|
194
216
|
|
|
195
217
|
**Other**:
|
|
196
|
-
* Remove `package-lock.json` from the repository
|
|
197
218
|
|
|
219
|
+
* Remove `package-lock.json` from the repository
|
|
198
220
|
|
|
199
221
|
## `0.5.x` (July, 19 2023)
|
|
200
222
|
|
|
201
223
|
**New features**:
|
|
202
224
|
|
|
203
225
|
**Enhancements**:
|
|
226
|
+
|
|
204
227
|
* Add large numbers of VITS voices
|
|
205
228
|
|
|
206
229
|
**Behavioral changes**:
|
|
230
|
+
|
|
207
231
|
* Move to new package system supporting version tags, and hosted in a Hugging Face repository
|
|
208
232
|
|
|
209
233
|
**Fixes**:
|
|
210
|
-
* Various fixes
|
|
211
234
|
|
|
235
|
+
* Various fixes
|
|
212
236
|
|
|
213
237
|
## `0.4.x` (July 9, 2023)
|
|
214
238
|
|
|
215
239
|
**New features**:
|
|
216
240
|
|
|
217
241
|
**Enhancements**:
|
|
242
|
+
|
|
218
243
|
* Improve text normalization and add support for currencies
|
|
219
244
|
* Improve heteronym lexicons
|
|
220
245
|
* Extend year patterns
|
|
@@ -223,17 +248,19 @@ Many features, enhancements, and fixes were incrementally added over the span of
|
|
|
223
248
|
* Add large numbers of VITS voices
|
|
224
249
|
|
|
225
250
|
**Behavioral changes**:
|
|
251
|
+
|
|
226
252
|
* Remove support for `afplay` and `aplay` for playback. Only SoX is used now.
|
|
227
253
|
|
|
228
254
|
**Fixes**:
|
|
229
|
-
* Various fixes
|
|
230
255
|
|
|
256
|
+
* Various fixes
|
|
231
257
|
|
|
232
258
|
## `0.3.x` (July 1, 2023)
|
|
233
259
|
|
|
234
260
|
**New features**:
|
|
235
261
|
|
|
236
262
|
**Enhancements**:
|
|
263
|
+
|
|
237
264
|
* Upgrade heteronym disambiguation to an improved, rule-based approach, which doesn't use POS tagging
|
|
238
265
|
* Extend decade normalization
|
|
239
266
|
* Improve text normalization and add support to currencies
|
|
@@ -242,15 +269,16 @@ Many features, enhancements, and fixes were incrementally added over the span of
|
|
|
242
269
|
**Behavioral changes**:
|
|
243
270
|
|
|
244
271
|
**Fixes**:
|
|
272
|
+
|
|
245
273
|
* Remove sentences containing only whitespace when synthesizing
|
|
246
274
|
* Many fixes
|
|
247
275
|
|
|
248
|
-
|
|
249
276
|
## `0.2.x` (May 10, 2023)
|
|
250
277
|
|
|
251
278
|
**New features**:
|
|
252
279
|
|
|
253
280
|
**Enhancements**:
|
|
281
|
+
|
|
254
282
|
* Show current sentence and segment in synthesis log
|
|
255
283
|
* Add check for cancellation flag
|
|
256
284
|
* Add support to additional VITS voices
|
|
@@ -258,20 +286,23 @@ Many features, enhancements, and fixes were incrementally added over the span of
|
|
|
258
286
|
* Many Enhancements
|
|
259
287
|
|
|
260
288
|
**Behavioral changes**:
|
|
289
|
+
|
|
261
290
|
* Remove dependency on `xregexp` package. Use Unicode RegExp instead.
|
|
262
291
|
|
|
263
292
|
**Fixes**:
|
|
264
|
-
* Many fixes
|
|
265
293
|
|
|
294
|
+
* Many fixes
|
|
266
295
|
|
|
267
296
|
## `0.1.x` (April 24, 2023)
|
|
268
297
|
|
|
269
298
|
Initial release
|
|
270
299
|
|
|
271
300
|
**Enhancements**:
|
|
301
|
+
|
|
272
302
|
* Add support for skipping audio playback with the Enter key.
|
|
273
303
|
* Run CLI in a worker thread by default.
|
|
274
304
|
* Many Enhancements
|
|
275
305
|
|
|
276
306
|
**Fixes**:
|
|
307
|
+
|
|
277
308
|
* Many fixes
|
package/docs/Server.md
CHANGED
|
@@ -27,19 +27,22 @@ For Node.js clients, a simple client class allows to wrap communications with th
|
|
|
27
27
|
Currently, the client is embedded in the main codebase. This means you have to import the `echogarden` package to use it:
|
|
28
28
|
|
|
29
29
|
```ts
|
|
30
|
-
import { WebSocket } from 'ws'
|
|
31
30
|
import { Client } from 'echogarden'
|
|
31
|
+
import { WebSocket } from 'ws'
|
|
32
32
|
|
|
33
33
|
const ws = new WebSocket('ws://localhost:45054')
|
|
34
34
|
|
|
35
|
-
ws.on(
|
|
35
|
+
ws.on('open', async () => {
|
|
36
36
|
const client = new Client(ws)
|
|
37
37
|
|
|
38
|
-
const { audio } = await client.synthesize(
|
|
38
|
+
const { audio } = await client.synthesize('Hello World', {
|
|
39
|
+
engine: 'espeak',
|
|
40
|
+
})
|
|
39
41
|
})
|
|
40
42
|
```
|
|
41
43
|
|
|
42
44
|
**TODO**:
|
|
45
|
+
|
|
43
46
|
* Separate the client to an independent, lightweight, Node.js package, with browser compatibility
|
|
44
47
|
* Add support for cancellation signals
|
|
45
48
|
* Document how to use with a background worker
|
|
@@ -72,6 +75,7 @@ The `messageType` property is a string representing the operation to perform. Th
|
|
|
72
75
|
When sending a message, `requestId` should contain a long random string that uniquely identifies your request, like `cb7e0f3ec835a213b005c4424c8d5775`.
|
|
73
76
|
|
|
74
77
|
For example, this message requests synthesis:
|
|
78
|
+
|
|
75
79
|
```ts
|
|
76
80
|
{
|
|
77
81
|
messageType: 'SynthesisRequest',
|
|
@@ -130,16 +134,21 @@ To cancel an existing request, the client can send a `CancellationRequest` messa
|
|
|
130
134
|
## Starting the server programmatically
|
|
131
135
|
|
|
132
136
|
You can use the `startServer` method to start a new server.
|
|
137
|
+
|
|
133
138
|
```ts
|
|
134
|
-
async function startServer(
|
|
139
|
+
async function startServer(
|
|
140
|
+
serverOptions: ServerOptions,
|
|
141
|
+
onStarted: (options: ServerOptions) => void,
|
|
142
|
+
)
|
|
135
143
|
```
|
|
136
144
|
|
|
137
145
|
Example:
|
|
146
|
+
|
|
138
147
|
```ts
|
|
139
148
|
import { startServer } from 'echogarden'
|
|
140
149
|
|
|
141
150
|
await startServer({ port: 1234 }, () => {
|
|
142
|
-
console.log(
|
|
151
|
+
console.log('Server is started!')
|
|
143
152
|
})
|
|
144
153
|
```
|
|
145
154
|
|
package/docs/Tasklist.md
CHANGED
|
@@ -11,10 +11,12 @@
|
|
|
11
11
|
* Option to disable single sentence per cue
|
|
12
12
|
|
|
13
13
|
### Browser extension
|
|
14
|
+
|
|
14
15
|
* Investigate why WebSpeech events sometimes completely stop working in the middle of an utterance for no apparent reason. Sometimes this is permanent, until the extension is restarted. Is this a browser issue?
|
|
15
16
|
* If a request is made and the server takes too much time to respond, the service worker may sleep and the request never canceled
|
|
16
17
|
|
|
17
18
|
### Browser extension / content script
|
|
19
|
+
|
|
18
20
|
* Highlighting sometimes does not appear when mouse is pressed over handle while speech of element starts
|
|
19
21
|
|
|
20
22
|
### External bugs
|
|
@@ -31,6 +33,7 @@
|
|
|
31
33
|
## Features and enhancements
|
|
32
34
|
|
|
33
35
|
### CLI
|
|
36
|
+
|
|
34
37
|
* Show names of files written to disk. This is useful for cases where a file is auto-renamed to prevent overwriting existing data
|
|
35
38
|
* Restrict input media file extensions to ensure that invalid files are not passed to FFmpeg
|
|
36
39
|
* Show a message when a new version is available
|
|
@@ -46,28 +49,35 @@
|
|
|
46
49
|
* Mode to print IPA words when speaking
|
|
47
50
|
|
|
48
51
|
### CLI / playback
|
|
52
|
+
|
|
49
53
|
* Option to set audio output device for playback
|
|
50
54
|
* Option to set playback volume
|
|
51
55
|
* Maybe find a way not to pre-normalize if the audio is silent (to prevent a 30dB increase of possible noise)
|
|
52
56
|
* Add phone playback support
|
|
53
57
|
|
|
54
58
|
### CLI / `speak`
|
|
59
|
+
|
|
55
60
|
* Add support for sentence templates, like `echogarden speak-file text.txt /parts/[sentence].wav`
|
|
56
61
|
|
|
57
62
|
### CLI / `speak-wikipedia`
|
|
63
|
+
|
|
58
64
|
* Correctly detect language when a Wikipedia URL is passed instead of an article name
|
|
59
65
|
* Add option to set language edition separately from language, since Wikipedia language editions has its own code system that is slightly different from the standard one, in some cases
|
|
60
66
|
|
|
61
67
|
### CLI / `speak-url`
|
|
68
|
+
|
|
62
69
|
* Use the Wikipedia reader when the URL is detected to be from `wikipedia.org`
|
|
63
70
|
|
|
64
71
|
### CLI / `list-voices`
|
|
72
|
+
|
|
65
73
|
* When given a configuration file, see if you can fall back to take options from `speak` options, for example, to take API keys that are required for both the synthesis request and voice list request and
|
|
66
74
|
|
|
67
75
|
### CLI / `list-packages`
|
|
76
|
+
|
|
68
77
|
* Support filters
|
|
69
78
|
|
|
70
79
|
### CLI / New commands
|
|
80
|
+
|
|
71
81
|
* `play-with-subtitles`: Preview subtitles in terminal
|
|
72
82
|
* `play-with-timeline`: Preview timeline in terminal
|
|
73
83
|
* `subtitles-to-text`, `subtitles-to-timeline`, `srt-to-vtt`, `vtt-to-srt`
|
|
@@ -78,21 +88,25 @@
|
|
|
78
88
|
* `speak-youtube-subtitles`: To speak the subtitles of a YouTube video
|
|
79
89
|
|
|
80
90
|
### API
|
|
91
|
+
|
|
81
92
|
* Allow callers from API to cancel a task via `AbortController` and `AbortSignal`
|
|
82
93
|
* Validate timelines to ensure timestamps are always increasing: no undefined timestamps, no negative timestamps, out-of-order timestamps, or timestamps over the duration of the audio. No sentences without words, etc. Missing or incorrect word offsets, etc.
|
|
83
94
|
* Add support for phrases in timelines
|
|
84
95
|
* Accept voice list caching options in `SynthesisOptions`
|
|
85
96
|
|
|
86
97
|
### Package manager
|
|
98
|
+
|
|
87
99
|
* Better error message when a package is not found remotely. Currently, it just gives a `404 not found` without any other information
|
|
88
100
|
* Retry on network failure
|
|
89
101
|
|
|
90
102
|
### Speech language detection
|
|
91
103
|
|
|
92
104
|
### Text language detection
|
|
105
|
+
|
|
93
106
|
* Deploy and add the new n-gram based text language detection model
|
|
94
107
|
|
|
95
108
|
### Subtitles
|
|
109
|
+
|
|
96
110
|
* Split long words if needed
|
|
97
111
|
* Decide how many punctuation characters to allow before breaking to a new line (currently it's infinite)
|
|
98
112
|
* If a subtitle is too short and at the end of the audio, try to extend it back if possible (for example, if the previous subtitle is already extended, take back from it)
|
|
@@ -101,6 +115,7 @@
|
|
|
101
115
|
* Parse VTT's language
|
|
102
116
|
|
|
103
117
|
### Synthesis
|
|
118
|
+
|
|
104
119
|
* Option to disable alignment (only for some engines). Alternative: use a low granularity DTW setting that is very fast to compute
|
|
105
120
|
* Find places to add commas (",") to improve speech fluency. VITS voices don't normally add speech breaks if there is no punctuation
|
|
106
121
|
* An isolated dash " - " can maybe be converted to a " , " to ensure there's a break in the speech
|
|
@@ -117,6 +132,7 @@
|
|
|
117
132
|
* When `splitToSentences` is set to `false`, the timeline doesn't include proper sentences. Find a way to pass larger sections to the TTS, but still have proper sentences in the timeline
|
|
118
133
|
|
|
119
134
|
### Synthesis / preprocessing
|
|
135
|
+
|
|
120
136
|
* Full date normalization (e.g. `21 August 2023`, `21 Aug 2023`, `August 21, 2023`)
|
|
121
137
|
* Add support for capitalized-only rules, and possibly also all uppercase / all lowercase rules
|
|
122
138
|
* Add support for multiple consecutive words in `precededBy` and `followedBy` conditions
|
|
@@ -130,6 +146,7 @@
|
|
|
130
146
|
* `≈` symbol as a word character
|
|
131
147
|
|
|
132
148
|
### Synthesis / VITS
|
|
149
|
+
|
|
133
150
|
* Consider adding `⦁︎` (when surrounded by whitespace) as phrase separator
|
|
134
151
|
* Allow limiting how many models are cached in memory.
|
|
135
152
|
* Ensure that caching behaves correctly when the same model is used with different execution providers
|
|
@@ -138,49 +155,60 @@
|
|
|
138
155
|
* Add speaker names to voice list somehow
|
|
139
156
|
|
|
140
157
|
### Synthesis / Kokoro
|
|
158
|
+
|
|
141
159
|
* Consider adding `⦁︎` (when surrounded by whitespace) as phrase separator
|
|
142
160
|
* Ensure that caching behaves correctly when the same model is used with different execution providers
|
|
143
161
|
|
|
144
162
|
### Synthesis / Azure Cognitive Services
|
|
163
|
+
|
|
145
164
|
* Currently, when input is set to be SSML, it is wrapped in a `<speak>` tag. Handle the case where the user made their own SSML document wrapped with a `<speak>` tag as well. Currently, it may send invalid input to Azure
|
|
146
165
|
|
|
147
166
|
### Recognition
|
|
167
|
+
|
|
148
168
|
* Add token UTF-8 bytes in token entries on timeline. Some tokens are only a part of a codepoint, so this will allow to know exactly what they contain - the string representation is would be a Unicode error symbol in that case
|
|
149
169
|
* Show alternatives when playing in the CLI. Clear current line and rewrite already printed text for alternatives during the speech recognition process
|
|
150
170
|
|
|
151
171
|
### Recognition / Whisper
|
|
172
|
+
|
|
152
173
|
* Whisper's Chinese and Japanese output can be split into words in a more accurate way. Consider using a dedicated segmentation library to perform the segmentation in character sequences that have no punctuation characters to aid on guessing word boundaries
|
|
153
|
-
* Cache last model (if enough memory is available). Ensure that caching works when switching between
|
|
174
|
+
* Cache last model (if enough memory is available). Ensure that caching works when switching between different execution providers
|
|
154
175
|
* Whisper timestamp tokens can be used to split into segments, otherwise it is possible to try to guess using pause lengths or voice activity detection
|
|
155
176
|
* Bring back the option to use eSpeak DTW based alignment on segments, as an alternative approach
|
|
156
177
|
|
|
157
178
|
### Alignment
|
|
158
179
|
|
|
159
180
|
### Alignment / DTW
|
|
181
|
+
|
|
160
182
|
* For the `granularity` option, add more granularities like `xxx-low` and `xxxx-low` (should the naming be changed? Maybe transition to a new naming scheme?)
|
|
161
183
|
* Add and test official support for more than 6 hours of audio
|
|
162
184
|
|
|
163
185
|
### Alignment / DTW-RA
|
|
164
186
|
|
|
165
187
|
### Alignment / Whisper
|
|
188
|
+
|
|
166
189
|
* Show same token stats as recognition now does on `trace` mode
|
|
167
190
|
|
|
168
191
|
### Source separation / MDX-NET
|
|
192
|
+
|
|
169
193
|
* Option to customize overlap
|
|
170
194
|
|
|
171
195
|
### Server
|
|
196
|
+
|
|
172
197
|
* Option to allow or disallow local file paths as arguments to API methods (as a security safeguard)
|
|
173
198
|
|
|
174
199
|
### Worker
|
|
200
|
+
|
|
175
201
|
* Add cancellation checks in more operations
|
|
176
202
|
* Support more operations
|
|
177
203
|
|
|
178
204
|
### Browser extension
|
|
205
|
+
|
|
179
206
|
* Options UI
|
|
180
207
|
* Add supported engines and voices to WebSpeech voice list
|
|
181
208
|
* Pause and resume support
|
|
182
209
|
|
|
183
210
|
### Browser extension / content script
|
|
211
|
+
|
|
184
212
|
* Autoscroll should work even if the scrollbar relevant to the target element is not the viewport's scrollbar
|
|
185
213
|
* Find a way to show handles even for elements that start with a link
|
|
186
214
|
* Add detection for line breaks in `pre` blocks
|
|
@@ -216,6 +244,7 @@
|
|
|
216
244
|
## Future features and enhancements
|
|
217
245
|
|
|
218
246
|
### CLI
|
|
247
|
+
|
|
219
248
|
* Auto-generate options file, with comments, based on default options of the API
|
|
220
249
|
* Have the CLI launch a background worker (in a thread) to enable better parallelism
|
|
221
250
|
* Playback result audio while synthesis or recognition is still processing in the background
|
|
@@ -225,23 +254,29 @@
|
|
|
225
254
|
* Markdown file as text input?
|
|
226
255
|
|
|
227
256
|
### OpenAI compatible local server
|
|
257
|
+
|
|
228
258
|
* `echogarden serve` would serve a OpenAI-compatible server for all speech recognition, speech translation and speech synthesis engines
|
|
229
259
|
|
|
230
260
|
### Web
|
|
261
|
+
|
|
231
262
|
* Web based frontend UI to the server
|
|
232
263
|
* Adapt some WASM modules to also run on the web
|
|
233
264
|
* Investigate running in WebContainer
|
|
234
265
|
|
|
235
266
|
### API
|
|
267
|
+
|
|
236
268
|
* Auto-install npm modules when needed using an approach similar to like `npm-programmatic`
|
|
237
269
|
|
|
238
270
|
### Text enhancement
|
|
271
|
+
|
|
239
272
|
* Add capitalization and punctuation to recognized outputs if needed (Silero has a model for it for `en`, `de`, `ru`, `es`, but in `.pt` format only)
|
|
240
273
|
|
|
241
274
|
### Synthesis
|
|
275
|
+
|
|
242
276
|
* Synthesize the given subtitle file and try to preserve the existing timing of cues, or even align to existing speech
|
|
243
277
|
|
|
244
278
|
### Recognition
|
|
279
|
+
|
|
245
280
|
* Low latency, streaming recognition mode. Make the partial transcription available as fast as possible
|
|
246
281
|
* Live input / microphone recognition
|
|
247
282
|
* Implement beam search for Whisper decoder
|
|
@@ -249,6 +284,7 @@
|
|
|
249
284
|
* Investigate exporting Whisper models to 16-bit quantized ONNX or a mix of 16-bit and 32-bit
|
|
250
285
|
|
|
251
286
|
### Alignment
|
|
287
|
+
|
|
252
288
|
* Method to align audio file to audio file
|
|
253
289
|
* Allow `dtw` mode work with more speech synthesizers to produce its reference
|
|
254
290
|
* Predict timing for individual letters (graphemes) based on phoneme timestamps (especially useful for Chinese and Japanese)
|
package/docs/Technical.md
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "echogarden",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.4.0",
|
|
4
4
|
"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "MIT AND GPL-3.0",
|
|
@@ -44,7 +44,8 @@
|
|
|
44
44
|
"tsconfig.json"
|
|
45
45
|
],
|
|
46
46
|
"scripts": {
|
|
47
|
-
"test": "
|
|
47
|
+
"test": "vitest",
|
|
48
|
+
"test-manual": "node --experimental-wasi-unstable-preview1 --no-warnings --trace-uncaught ./dist/tests/Test.js",
|
|
48
49
|
"generate-options-schema": "npx ts-json-schema-generator --path ./src/api/APIOptions.ts --unstable --type APIOptions --tsconfig ./tsconfig.json --out ./data/schemas/options.json",
|
|
49
50
|
"make-tarballs": "node --no-warnings ./dist/build-tools/MakeTarballsForInstalledPackages.js"
|
|
50
51
|
},
|
|
@@ -52,8 +53,8 @@
|
|
|
52
53
|
"echogarden": "./dist/cli/CLILauncher.js"
|
|
53
54
|
},
|
|
54
55
|
"dependencies": {
|
|
55
|
-
"@aws-sdk/client-polly": "~3.
|
|
56
|
-
"@aws-sdk/client-transcribe-streaming": "~3.
|
|
56
|
+
"@aws-sdk/client-polly": "~3.1127.0",
|
|
57
|
+
"@aws-sdk/client-transcribe-streaming": "~3.1127.0",
|
|
57
58
|
"@echogarden/audio-io": "~0.4.2",
|
|
58
59
|
"@echogarden/fasttext-wasm": "~0.1.0",
|
|
59
60
|
"@echogarden/flite-wasi": "~0.1.1",
|
|
@@ -64,7 +65,7 @@
|
|
|
64
65
|
"@echogarden/sonic-wasm": "~0.2.0",
|
|
65
66
|
"@echogarden/speex-resampler-wasm": "~0.3.0",
|
|
66
67
|
"@echogarden/svoxpico-wasm": "~0.2.0",
|
|
67
|
-
"@echogarden/text-segmentation": "~0.
|
|
68
|
+
"@echogarden/text-segmentation": "~0.8.1",
|
|
68
69
|
"@echogarden/transformers-nodejs-lite": "~2.17.1-lite.4",
|
|
69
70
|
"@echogarden/wave-codec": "~0.4.0",
|
|
70
71
|
"@echogarden/whisper.cpp-binding": "~0.2.2",
|
|
@@ -74,25 +75,25 @@
|
|
|
74
75
|
"easier-http-request": "~0.1.0",
|
|
75
76
|
"fs-extra": "~11.4.0",
|
|
76
77
|
"graceful-fs": "~4.2.11",
|
|
77
|
-
"html-to-text": "~10.0.
|
|
78
|
+
"html-to-text": "~10.0.1",
|
|
78
79
|
"import-meta-resolve": "~4.2.0",
|
|
79
80
|
"jieba-wasm": "~2.4.0",
|
|
80
81
|
"jsdom": "~30.0.1",
|
|
81
82
|
"kuromoji": "~0.1.2",
|
|
82
83
|
"microsoft-cognitiveservices-speech-sdk": "~1.51.0",
|
|
83
|
-
"msgpack-lite": "~0.
|
|
84
|
+
"msgpack-lite": "~0.2.2",
|
|
84
85
|
"onnxruntime-node": "~1.21.1",
|
|
85
|
-
"openai": "~7.
|
|
86
|
-
"quick-json5": "~0.
|
|
87
|
-
"regexp-composer": "~0.
|
|
86
|
+
"openai": "~7.10.0",
|
|
87
|
+
"quick-json5": "~0.5.0",
|
|
88
|
+
"regexp-composer": "~0.7.0",
|
|
88
89
|
"sam-js": "~0.3.1",
|
|
89
90
|
"standard-html-escaper": "~0.2.1",
|
|
90
91
|
"strip-ansi": "~7.2.0",
|
|
91
92
|
"tar": "~7.5.22",
|
|
92
93
|
"tiktoken": "~1.0.22",
|
|
93
94
|
"tinyld": "~1.3.4",
|
|
94
|
-
"wasm-feature-detect": "~1.
|
|
95
|
-
"wasm-heap-manager": "~0.
|
|
95
|
+
"wasm-feature-detect": "~1.9.0",
|
|
96
|
+
"wasm-heap-manager": "~0.5.1",
|
|
96
97
|
"ws": "~8.21.3",
|
|
97
98
|
"wtf_wikipedia": "~10.4.2"
|
|
98
99
|
},
|
|
@@ -118,10 +119,11 @@
|
|
|
118
119
|
"@types/graceful-fs": "~4.1.9",
|
|
119
120
|
"@types/jsdom": "~30.0.0",
|
|
120
121
|
"@types/msgpack-lite": "~0.1.12",
|
|
121
|
-
"@types/node": "~26.
|
|
122
|
+
"@types/node": "~26.5.0",
|
|
122
123
|
"@types/ws": "~8.18.1",
|
|
123
124
|
"ts-json-schema-generator": "~2.9.0",
|
|
124
|
-
"typescript": "~7.0.2"
|
|
125
|
+
"typescript": "~7.0.2",
|
|
126
|
+
"vitest": "~5.0.0"
|
|
125
127
|
},
|
|
126
128
|
"overrides": {
|
|
127
129
|
"whatwg-url": "~14.0.0",
|