echogarden 3.2.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +5 -0
  2. package/dist/api/Synthesis.d.ts.map +1 -1
  3. package/dist/api/Synthesis.js +31 -21
  4. package/dist/api/Synthesis.js.map +1 -1
  5. package/dist/nlp/TextNormalizer.js +1 -1
  6. package/dist/nlp/TextNormalizer.js.map +1 -1
  7. package/dist/synthesis/EspeakTTS.d.ts.map +1 -1
  8. package/dist/synthesis/EspeakTTS.js +20 -5
  9. package/dist/synthesis/EspeakTTS.js.map +1 -1
  10. package/dist/utilities/Timer.d.ts +7 -3
  11. package/dist/utilities/Timer.d.ts.map +1 -1
  12. package/dist/utilities/Timer.js +42 -30
  13. package/dist/utilities/Timer.js.map +1 -1
  14. package/docs/API.md +51 -34
  15. package/docs/CLI.md +41 -6
  16. package/docs/Contributing.md +1 -1
  17. package/docs/Development.md +22 -18
  18. package/docs/Engines.md +0 -1
  19. package/docs/Licenses.md +2 -0
  20. package/docs/Options.md +56 -4
  21. package/docs/Releases.md +45 -14
  22. package/docs/Server.md +14 -5
  23. package/docs/Tasklist.md +37 -1
  24. package/docs/Technical.md +1 -0
  25. package/package.json +16 -14
  26. package/src/api/Synthesis.ts +39 -27
  27. package/src/nlp/TextNormalizer.ts +1 -2
  28. package/src/synthesis/EspeakTTS.ts +21 -6
  29. package/src/utilities/Timer.ts +57 -37
  30. package/tsconfig.json +51 -53
  31. package/dist/encodings/HtmlEscape.d.ts +0 -2
  32. package/dist/encodings/HtmlEscape.d.ts.map +0 -1
  33. package/dist/encodings/HtmlEscape.js +0 -30
  34. package/dist/encodings/HtmlEscape.js.map +0 -1
  35. package/dist/encodings/LEB128.d.ts +0 -2
  36. package/dist/encodings/LEB128.d.ts.map +0 -1
  37. package/dist/encodings/LEB128.js +0 -2
  38. package/dist/encodings/LEB128.js.map +0 -1
  39. package/dist/utilities/StringBuilder.d.ts +0 -11
  40. package/dist/utilities/StringBuilder.d.ts.map +0 -1
  41. package/dist/utilities/StringBuilder.js +0 -39
  42. package/dist/utilities/StringBuilder.js.map +0 -1
package/docs/Releases.md CHANGED
@@ -7,6 +7,7 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
7
7
  ## `1.0.0` (April 12, 2024)
8
8
 
9
9
  **New features**:
10
+
10
11
  * Add support for [`whisper.cpp`](https://github.com/ggerganov/whisper.cpp), a C++ port of OpenAI's Whisper speech recognition architecture. It is faster than the integrated `whisper` engine, supports large models, and GPU processing. It can now be used for recognition, speech translation and alignment (via `dtw-ra`), though its word timestamps are less accurate than the integrated `whisper` engine.
11
12
  * Add the [MDX-NET](https://github.com/kuielab/mdx-net/) source separation model, enabling vocal tracks to be extracted from music and speech audio using the new `isolate` operation
12
13
  * Add support for OpenAI cloud platform's [speech recognition and translation services](https://platform.openai.com/docs/guides/speech-to-text)
@@ -18,17 +19,19 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
18
19
  * Expose more options for the Elevenlabs engine
19
20
 
20
21
  **Enhancements**:
22
+
21
23
  * `detect-voice-activity` now also outputs the cropped voice when an audio file is given as output, with `.cropped` suffix added
22
24
  * Improved word segmentation of whisper engine. Words spanning multiple recognized time segments are now split. Word timestamp ranges don't overlap non-speech sections.
23
25
  * Whisper and Whisper.cpp outputs now include timestamps for individual recognized tokens
24
26
 
25
27
  **Behavioral and breaking changes**:
28
+
26
29
  * Minimal required node version changed to `18.0.0`
27
30
  * All recognition, alignment, translation and language recognition operations first apply the new adaptive gate VAD (can be changed to any other VAD engine via the `vad.` option prefix) and remove any sections that are not identified as containing voice, before starting processing. This should improve results in most cases, and reduce processing time
28
31
  * To reduce Whisper hallucinations and repetition loops, these changes were made:
29
- * Pre-cropping by default can significantly help with reducing hallucinations, giving the model less "empty space" to hallucinate on
30
- * Enable new token-level repetition suppression (`suppressRepetition = true`) during decoding
31
- * Disable decoding of timestamp tokens by default (`decodeTimestampTokens = false`), since more accurate timing is already extracted via cross-attention weight alignment. For unclear reasons, this can significantly reduce the occurrence of token repetition loops, and increases word timestamp accuracy. However, there are cases where this causes the model to end a part prematurely, especially in singing and less speech-like voice segments. In those cases the it can be enabled with `decodeTimestampTokens = true`
32
+ * Pre-cropping by default can significantly help with reducing hallucinations, giving the model less "empty space" to hallucinate on
33
+ * Enable new token-level repetition suppression (`suppressRepetition = true`) during decoding
34
+ * Disable decoding of timestamp tokens by default (`decodeTimestampTokens = false`), since more accurate timing is already extracted via cross-attention weight alignment. For unclear reasons, this can significantly reduce the occurrence of token repetition loops, and increases word timestamp accuracy. However, there are cases where this causes the model to end a part prematurely, especially in singing and less speech-like voice segments. In those cases the it can be enabled with `decodeTimestampTokens = true`
32
35
  * When `transcribe`, `align` or `translate-speech` operations are run with `--isolate` enabled, they will output the isolated part in `some-output-file.isolated.wav` and background part (isolated subtracted from original) in `some-output-file.background.wav` (any supported codec other than `wav` can be used - this is just an example)
33
36
  * VAD operations now return a timeline including only the active sections, labeled as `active`
34
37
  * When the specified language is not English, but an `.en` Whisper model was specified, a warning would be shown, and the model will be automatically switched to the corresponding multilingual model (omitting the `.en`), instead of producing an error
@@ -40,6 +43,7 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
40
43
  * `dryMixGainDb` denoising option renamed to `dryMixGain`
41
44
 
42
45
  **Fixes**:
46
+
43
47
  * Integrated `whisper` engine now uses `tiktoken` to tokenize text, which produces near identical tokenization compared to the official Python implementation. This fixes issues with decoding Chinese characters, correctly encoding prompts, and several other languages with characters that span multiple tokens
44
48
  * Fix voice language lists for multilingual voices in Elevenlabs TTS engine
45
49
  * Fix runtime error with `rnnoise` when used as VAD engine
@@ -53,6 +57,7 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
53
57
  * Various other fixes
54
58
 
55
59
  **Documentation**:
60
+
56
61
  * Add new [release notes](./Releases.md) page. Releases before `1.0.x` were retroactively documented based on commit history (may not exactly detail all historical changes)
57
62
  * Add new [development page](./Development.md) with a quick guide for setting up a development environment to make changes to the Echogarden codebase
58
63
  * Options reference page reorganized and updated with some missing information
@@ -60,24 +65,27 @@ For releases after `1.0.0`, see the [GitHub releases page](https://github.com/ec
60
65
  ## `0.12.x` (March 16, 2024)
61
66
 
62
67
  **New features**:
68
+
63
69
  * Add global options support to API
64
70
  * Add updated `ffmpeg` internal packages for many platforms, including pre-signed binaries for macOS
65
71
 
66
72
  **Enhancements**:
67
73
 
68
74
  **Fixes**:
75
+
69
76
  * Fix issue with Polish TTS in eSpeak NG
70
77
  * Fix warning about `punycode` module in the CLI
71
78
 
72
79
  **Other**:
73
- * Include `package-lock.json` in repository
74
80
 
81
+ * Include `package-lock.json` in repository
75
82
 
76
83
  ## `0.11.x` (August 17, 2023)
77
84
 
78
85
  Many features, enhancements, and fixes were incrementally added over the span of 7 months, up to March 2024.
79
86
 
80
87
  **New features**:
88
+
81
89
  * Partial rewrite of subtitle generation methods. Adds many features and options
82
90
  * Expose subtitle methods to API
83
91
  * Add support for multi-pass (AKA hierarchical) DTW alignment
@@ -85,72 +93,82 @@ Many features, enhancements, and fixes were incrementally added over the span of
85
93
  * Add support for Klatt synthesis in `espeak` engine
86
94
 
87
95
  **Enhancements**:
96
+
88
97
  * Adds a total of 14 new VITS voices
89
98
  * Retry on failure for `microsoft-edge` and `google-translate` TTS engines
90
99
 
91
100
  **Fixes**:
101
+
92
102
  * Don't error when empty audio is returned in `microsoft-edge` response
93
103
  * Fix audio playback in macOS
94
104
  * Many other fixes
95
105
 
96
-
97
106
  ## `0.10.x` (August 2, 2023)
98
107
 
99
108
  **New features**:
109
+
100
110
  * Add developer and server APIs, allowing the package to be used as a library or server
101
111
  * Add granularity options to DTW
102
112
  * Add Linux SoX package
103
113
 
104
114
  **Enhancements**:
115
+
105
116
  * Adds 7 new VITS voices
106
117
 
107
118
  **Fixes**:
108
- * Many fixes
109
119
 
120
+ * Many fixes
110
121
 
111
122
  ## `0.9.x` (July 29, 2023)
112
123
 
113
124
  **New features**:
125
+
114
126
  * Add text offsets to timeline
115
127
  * Include segments and sentences in recognition and speech translation timelines
116
128
  * Improve voice information in Elevenlabs engine
117
129
 
118
130
  **Behavioral changes**:
131
+
119
132
  * Set `en_GB-alan-low` as default `en-GB` VITS voice, since `danny` was trained to accept `en-US` pronunciations, which may confuse some people
120
133
 
121
134
  **Fixes**:
135
+
122
136
  * Work around several eSpeak bugs
123
137
  * Many fixes
124
138
 
125
-
126
139
  ## `0.8.x` (July 25, 2023)
127
140
 
128
141
  **New features**:
142
+
129
143
  * Add confidence to some speech recognition timelines
130
144
  * Add `plaintext` options
131
145
  * Add punctuation thresholds to Whisper decoder
132
146
  * Expose subtitle configuration options to CLI
133
147
 
134
148
  **Enhancements**:
149
+
135
150
  * Improve speech language detection to work with arbitrary length audio. Split audio into overlapping parts and detect each part individually. Then average the results.
136
151
  * Add more heteronyms
137
152
 
138
153
  **Behavioral changes**:
154
+
139
155
  * Rename `subtitles.minWords` to `subtitles.minWordsInLine`
140
156
 
141
157
  **Fixes**:
158
+
142
159
  * Improve numerical stability of softmax function
143
160
  * Work around eSpeak bug with markers in long inputs
144
161
  * Various fixes
145
162
 
146
-
147
163
  ## `0.7.x` (July 23, 2023)
148
164
 
149
165
  **New features**:
166
+
150
167
  * Alignment: Add option to accept custom lexicons
151
168
  * Implement and add decoder temperature option to Whisper model
152
169
 
153
170
  **Enhancements**:
171
+
154
172
  * Alignment: use preprocessing and lexicons
155
173
  * Add language detection to speech translation
156
174
  * Add language detection options to synthesis, and set empty default options for detection in several APIs
@@ -162,13 +180,14 @@ Many features, enhancements, and fixes were incrementally added over the span of
162
180
  **Behavioral changes**:
163
181
 
164
182
  **Fixes**:
183
+
165
184
  * Fix support for SSML input in eSpeak engine
166
185
  * Various fixes
167
186
 
168
-
169
187
  ## `0.6.x` (July 20, 2023)
170
188
 
171
189
  **New features**:
190
+
172
191
  * Add support for custom lexicons. Change lexicon object structure to include language code and allow for multiple languages in a single lexicon
173
192
  * Add support for SSML inputs (currently only supported by Google, Microsoft and Amazon cloud engines). Ensure they are not split to segments or sentences
174
193
  * CLI: Add flag to enable or disable file overwriting in CLI
@@ -177,44 +196,50 @@ Many features, enhancements, and fixes were incrementally added over the span of
177
196
  * Change whitespace option to include option for collapsing all whitespace
178
197
 
179
198
  **Enhancements**:
199
+
180
200
  * Change whitespace option to include option for collapsing all whitespace
181
201
  * Add awareness of guillemets
182
202
  * Log full language of selected voice
183
203
 
184
204
  **Behavioral changes**:
205
+
185
206
  * Set Elevenlabs defaults to mid-values
186
207
  * Split plain text to paragraphs using double line breaks by default.
187
208
  * Rename `awsPolly` options to `amazonPolly` to be more consistent with documentation
188
209
 
189
210
  **Fixes**:
211
+
190
212
  * Fix and update Elevenlabs engine
191
213
  * Fix incorrect gender properties for some VITS voices
192
214
  * Convert to plaintext before detecting language when input is SSML.
193
215
  * Various fixes
194
216
 
195
217
  **Other**:
196
- * Remove `package-lock.json` from the repository
197
218
 
219
+ * Remove `package-lock.json` from the repository
198
220
 
199
221
  ## `0.5.x` (July, 19 2023)
200
222
 
201
223
  **New features**:
202
224
 
203
225
  **Enhancements**:
226
+
204
227
  * Add large numbers of VITS voices
205
228
 
206
229
  **Behavioral changes**:
230
+
207
231
  * Move to new package system supporting version tags, and hosted in a Hugging Face repository
208
232
 
209
233
  **Fixes**:
210
- * Various fixes
211
234
 
235
+ * Various fixes
212
236
 
213
237
  ## `0.4.x` (July 9, 2023)
214
238
 
215
239
  **New features**:
216
240
 
217
241
  **Enhancements**:
242
+
218
243
  * Improve text normalization and add support for currencies
219
244
  * Improve heteronym lexicons
220
245
  * Extend year patterns
@@ -223,17 +248,19 @@ Many features, enhancements, and fixes were incrementally added over the span of
223
248
  * Add large numbers of VITS voices
224
249
 
225
250
  **Behavioral changes**:
251
+
226
252
  * Remove support for `afplay` and `aplay` for playback. Only SoX is used now.
227
253
 
228
254
  **Fixes**:
229
- * Various fixes
230
255
 
256
+ * Various fixes
231
257
 
232
258
  ## `0.3.x` (July 1, 2023)
233
259
 
234
260
  **New features**:
235
261
 
236
262
  **Enhancements**:
263
+
237
264
  * Upgrade heteronym disambiguation to an improved, rule-based approach, which doesn't use POS tagging
238
265
  * Extend decade normalization
239
266
  * Improve text normalization and add support to currencies
@@ -242,15 +269,16 @@ Many features, enhancements, and fixes were incrementally added over the span of
242
269
  **Behavioral changes**:
243
270
 
244
271
  **Fixes**:
272
+
245
273
  * Remove sentences containing only whitespace when synthesizing
246
274
  * Many fixes
247
275
 
248
-
249
276
  ## `0.2.x` (May 10, 2023)
250
277
 
251
278
  **New features**:
252
279
 
253
280
  **Enhancements**:
281
+
254
282
  * Show current sentence and segment in synthesis log
255
283
  * Add check for cancellation flag
256
284
  * Add support to additional VITS voices
@@ -258,20 +286,23 @@ Many features, enhancements, and fixes were incrementally added over the span of
258
286
  * Many Enhancements
259
287
 
260
288
  **Behavioral changes**:
289
+
261
290
  * Remove dependency on `xregexp` package. Use Unicode RegExp instead.
262
291
 
263
292
  **Fixes**:
264
- * Many fixes
265
293
 
294
+ * Many fixes
266
295
 
267
296
  ## `0.1.x` (April 24, 2023)
268
297
 
269
298
  Initial release
270
299
 
271
300
  **Enhancements**:
301
+
272
302
  * Add support for skipping audio playback with the Enter key.
273
303
  * Run CLI in a worker thread by default.
274
304
  * Many Enhancements
275
305
 
276
306
  **Fixes**:
307
+
277
308
  * Many fixes
package/docs/Server.md CHANGED
@@ -27,19 +27,22 @@ For Node.js clients, a simple client class allows to wrap communications with th
27
27
  Currently, the client is embedded in the main codebase. This means you have to import the `echogarden` package to use it:
28
28
 
29
29
  ```ts
30
- import { WebSocket } from 'ws'
31
30
  import { Client } from 'echogarden'
31
+ import { WebSocket } from 'ws'
32
32
 
33
33
  const ws = new WebSocket('ws://localhost:45054')
34
34
 
35
- ws.on("open", async () => {
35
+ ws.on('open', async () => {
36
36
  const client = new Client(ws)
37
37
 
38
- const { audio } = await client.synthesize("Hello World", { engine: 'espeak' })
38
+ const { audio } = await client.synthesize('Hello World', {
39
+ engine: 'espeak',
40
+ })
39
41
  })
40
42
  ```
41
43
 
42
44
  **TODO**:
45
+
43
46
  * Separate the client to an independent, lightweight, Node.js package, with browser compatibility
44
47
  * Add support for cancellation signals
45
48
  * Document how to use with a background worker
@@ -72,6 +75,7 @@ The `messageType` property is a string representing the operation to perform. Th
72
75
  When sending a message, `requestId` should contain a long random string that uniquely identifies your request, like `cb7e0f3ec835a213b005c4424c8d5775`.
73
76
 
74
77
  For example, this message requests synthesis:
78
+
75
79
  ```ts
76
80
  {
77
81
  messageType: 'SynthesisRequest',
@@ -130,16 +134,21 @@ To cancel an existing request, the client can send a `CancellationRequest` messa
130
134
  ## Starting the server programmatically
131
135
 
132
136
  You can use the `startServer` method to start a new server.
137
+
133
138
  ```ts
134
- async function startServer(serverOptions: ServerOptions, onStarted: (options: ServerOptions) => void)
139
+ async function startServer(
140
+ serverOptions: ServerOptions,
141
+ onStarted: (options: ServerOptions) => void,
142
+ )
135
143
  ```
136
144
 
137
145
  Example:
146
+
138
147
  ```ts
139
148
  import { startServer } from 'echogarden'
140
149
 
141
150
  await startServer({ port: 1234 }, () => {
142
- console.log("Server is started!")
151
+ console.log('Server is started!')
143
152
  })
144
153
  ```
145
154
 
package/docs/Tasklist.md CHANGED
@@ -11,10 +11,12 @@
11
11
  * Option to disable single sentence per cue
12
12
 
13
13
  ### Browser extension
14
+
14
15
  * Investigate why WebSpeech events sometimes completely stop working in the middle of an utterance for no apparent reason. Sometimes this is permanent, until the extension is restarted. Is this a browser issue?
15
16
  * If a request is made and the server takes too much time to respond, the service worker may sleep and the request never canceled
16
17
 
17
18
  ### Browser extension / content script
19
+
18
20
  * Highlighting sometimes does not appear when mouse is pressed over handle while speech of element starts
19
21
 
20
22
  ### External bugs
@@ -31,6 +33,7 @@
31
33
  ## Features and enhancements
32
34
 
33
35
  ### CLI
36
+
34
37
  * Show names of files written to disk. This is useful for cases where a file is auto-renamed to prevent overwriting existing data
35
38
  * Restrict input media file extensions to ensure that invalid files are not passed to FFmpeg
36
39
  * Show a message when a new version is available
@@ -46,28 +49,35 @@
46
49
  * Mode to print IPA words when speaking
47
50
 
48
51
  ### CLI / playback
52
+
49
53
  * Option to set audio output device for playback
50
54
  * Option to set playback volume
51
55
  * Maybe find a way not to pre-normalize if the audio is silent (to prevent a 30dB increase of possible noise)
52
56
  * Add phone playback support
53
57
 
54
58
  ### CLI / `speak`
59
+
55
60
  * Add support for sentence templates, like `echogarden speak-file text.txt /parts/[sentence].wav`
56
61
 
57
62
  ### CLI / `speak-wikipedia`
63
+
58
64
  * Correctly detect language when a Wikipedia URL is passed instead of an article name
59
65
  * Add option to set language edition separately from language, since Wikipedia language editions has its own code system that is slightly different from the standard one, in some cases
60
66
 
61
67
  ### CLI / `speak-url`
68
+
62
69
  * Use the Wikipedia reader when the URL is detected to be from `wikipedia.org`
63
70
 
64
71
  ### CLI / `list-voices`
72
+
65
73
  * When given a configuration file, see if you can fall back to take options from `speak` options, for example, to take API keys that are required for both the synthesis request and voice list request and
66
74
 
67
75
  ### CLI / `list-packages`
76
+
68
77
  * Support filters
69
78
 
70
79
  ### CLI / New commands
80
+
71
81
  * `play-with-subtitles`: Preview subtitles in terminal
72
82
  * `play-with-timeline`: Preview timeline in terminal
73
83
  * `subtitles-to-text`, `subtitles-to-timeline`, `srt-to-vtt`, `vtt-to-srt`
@@ -78,21 +88,25 @@
78
88
  * `speak-youtube-subtitles`: To speak the subtitles of a YouTube video
79
89
 
80
90
  ### API
91
+
81
92
  * Allow callers from API to cancel a task via `AbortController` and `AbortSignal`
82
93
  * Validate timelines to ensure timestamps are always increasing: no undefined timestamps, no negative timestamps, out-of-order timestamps, or timestamps over the duration of the audio. No sentences without words, etc. Missing or incorrect word offsets, etc.
83
94
  * Add support for phrases in timelines
84
95
  * Accept voice list caching options in `SynthesisOptions`
85
96
 
86
97
  ### Package manager
98
+
87
99
  * Better error message when a package is not found remotely. Currently, it just gives a `404 not found` without any other information
88
100
  * Retry on network failure
89
101
 
90
102
  ### Speech language detection
91
103
 
92
104
  ### Text language detection
105
+
93
106
  * Deploy and add the new n-gram based text language detection model
94
107
 
95
108
  ### Subtitles
109
+
96
110
  * Split long words if needed
97
111
  * Decide how many punctuation characters to allow before breaking to a new line (currently it's infinite)
98
112
  * If a subtitle is too short and at the end of the audio, try to extend it back if possible (for example, if the previous subtitle is already extended, take back from it)
@@ -101,6 +115,7 @@
101
115
  * Parse VTT's language
102
116
 
103
117
  ### Synthesis
118
+
104
119
  * Option to disable alignment (only for some engines). Alternative: use a low granularity DTW setting that is very fast to compute
105
120
  * Find places to add commas (",") to improve speech fluency. VITS voices don't normally add speech breaks if there is no punctuation
106
121
  * An isolated dash " - " can maybe be converted to a " , " to ensure there's a break in the speech
@@ -117,6 +132,7 @@
117
132
  * When `splitToSentences` is set to `false`, the timeline doesn't include proper sentences. Find a way to pass larger sections to the TTS, but still have proper sentences in the timeline
118
133
 
119
134
  ### Synthesis / preprocessing
135
+
120
136
  * Full date normalization (e.g. `21 August 2023`, `21 Aug 2023`, `August 21, 2023`)
121
137
  * Add support for capitalized-only rules, and possibly also all uppercase / all lowercase rules
122
138
  * Add support for multiple consecutive words in `precededBy` and `followedBy` conditions
@@ -130,6 +146,7 @@
130
146
  * `≈` symbol as a word character
131
147
 
132
148
  ### Synthesis / VITS
149
+
133
150
  * Consider adding `⦁︎` (when surrounded by whitespace) as phrase separator
134
151
  * Allow limiting how many models are cached in memory.
135
152
  * Ensure that caching behaves correctly when the same model is used with different execution providers
@@ -138,49 +155,60 @@
138
155
  * Add speaker names to voice list somehow
139
156
 
140
157
  ### Synthesis / Kokoro
158
+
141
159
  * Consider adding `⦁︎` (when surrounded by whitespace) as phrase separator
142
160
  * Ensure that caching behaves correctly when the same model is used with different execution providers
143
161
 
144
162
  ### Synthesis / Azure Cognitive Services
163
+
145
164
  * Currently, when input is set to be SSML, it is wrapped in a `<speak>` tag. Handle the case where the user made their own SSML document wrapped with a `<speak>` tag as well. Currently, it may send invalid input to Azure
146
165
 
147
166
  ### Recognition
167
+
148
168
  * Add token UTF-8 bytes in token entries on timeline. Some tokens are only a part of a codepoint, so this will allow to know exactly what they contain - the string representation is would be a Unicode error symbol in that case
149
169
  * Show alternatives when playing in the CLI. Clear current line and rewrite already printed text for alternatives during the speech recognition process
150
170
 
151
171
  ### Recognition / Whisper
172
+
152
173
  * Whisper's Chinese and Japanese output can be split into words in a more accurate way. Consider using a dedicated segmentation library to perform the segmentation in character sequences that have no punctuation characters to aid on guessing word boundaries
153
- * Cache last model (if enough memory is available). Ensure that caching works when switching between different execution providers
174
+ * Cache last model (if enough memory is available). Ensure that caching works when switching between different execution providers
154
175
  * Whisper timestamp tokens can be used to split into segments, otherwise it is possible to try to guess using pause lengths or voice activity detection
155
176
  * Bring back the option to use eSpeak DTW based alignment on segments, as an alternative approach
156
177
 
157
178
  ### Alignment
158
179
 
159
180
  ### Alignment / DTW
181
+
160
182
  * For the `granularity` option, add more granularities like `xxx-low` and `xxxx-low` (should the naming be changed? Maybe transition to a new naming scheme?)
161
183
  * Add and test official support for more than 6 hours of audio
162
184
 
163
185
  ### Alignment / DTW-RA
164
186
 
165
187
  ### Alignment / Whisper
188
+
166
189
  * Show same token stats as recognition now does on `trace` mode
167
190
 
168
191
  ### Source separation / MDX-NET
192
+
169
193
  * Option to customize overlap
170
194
 
171
195
  ### Server
196
+
172
197
  * Option to allow or disallow local file paths as arguments to API methods (as a security safeguard)
173
198
 
174
199
  ### Worker
200
+
175
201
  * Add cancellation checks in more operations
176
202
  * Support more operations
177
203
 
178
204
  ### Browser extension
205
+
179
206
  * Options UI
180
207
  * Add supported engines and voices to WebSpeech voice list
181
208
  * Pause and resume support
182
209
 
183
210
  ### Browser extension / content script
211
+
184
212
  * Autoscroll should work even if the scrollbar relevant to the target element is not the viewport's scrollbar
185
213
  * Find a way to show handles even for elements that start with a link
186
214
  * Add detection for line breaks in `pre` blocks
@@ -216,6 +244,7 @@
216
244
  ## Future features and enhancements
217
245
 
218
246
  ### CLI
247
+
219
248
  * Auto-generate options file, with comments, based on default options of the API
220
249
  * Have the CLI launch a background worker (in a thread) to enable better parallelism
221
250
  * Playback result audio while synthesis or recognition is still processing in the background
@@ -225,23 +254,29 @@
225
254
  * Markdown file as text input?
226
255
 
227
256
  ### OpenAI compatible local server
257
+
228
258
  * `echogarden serve` would serve a OpenAI-compatible server for all speech recognition, speech translation and speech synthesis engines
229
259
 
230
260
  ### Web
261
+
231
262
  * Web based frontend UI to the server
232
263
  * Adapt some WASM modules to also run on the web
233
264
  * Investigate running in WebContainer
234
265
 
235
266
  ### API
267
+
236
268
  * Auto-install npm modules when needed using an approach similar to like `npm-programmatic`
237
269
 
238
270
  ### Text enhancement
271
+
239
272
  * Add capitalization and punctuation to recognized outputs if needed (Silero has a model for it for `en`, `de`, `ru`, `es`, but in `.pt` format only)
240
273
 
241
274
  ### Synthesis
275
+
242
276
  * Synthesize the given subtitle file and try to preserve the existing timing of cues, or even align to existing speech
243
277
 
244
278
  ### Recognition
279
+
245
280
  * Low latency, streaming recognition mode. Make the partial transcription available as fast as possible
246
281
  * Live input / microphone recognition
247
282
  * Implement beam search for Whisper decoder
@@ -249,6 +284,7 @@
249
284
  * Investigate exporting Whisper models to 16-bit quantized ONNX or a mix of 16-bit and 32-bit
250
285
 
251
286
  ### Alignment
287
+
252
288
  * Method to align audio file to audio file
253
289
  * Allow `dtw` mode work with more speech synthesizers to produce its reference
254
290
  * Predict timing for individual letters (graphemes) based on phoneme timestamps (especially useful for Chinese and Japanese)
package/docs/Technical.md CHANGED
@@ -18,6 +18,7 @@ Packages are downloaded as `.tar.gz` files, and are extracted to `[data-folder]/
18
18
  * `/home/user/.local/share/echogarden` on Linux
19
19
 
20
20
  `[package-id-string]` is formatted as:
21
+
21
22
  ```
22
23
  [engine name]-[package id]-[date as yyyymmdd]
23
24
  ```
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "echogarden",
3
- "version": "3.2.0",
3
+ "version": "3.4.0",
4
4
  "description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
5
5
  "author": "Rotem Dan",
6
6
  "license": "MIT AND GPL-3.0",
@@ -44,7 +44,8 @@
44
44
  "tsconfig.json"
45
45
  ],
46
46
  "scripts": {
47
- "test": "node --experimental-wasi-unstable-preview1 --no-warnings --trace-uncaught ./dist/tests/Test.js",
47
+ "test": "vitest",
48
+ "test-manual": "node --experimental-wasi-unstable-preview1 --no-warnings --trace-uncaught ./dist/tests/Test.js",
48
49
  "generate-options-schema": "npx ts-json-schema-generator --path ./src/api/APIOptions.ts --unstable --type APIOptions --tsconfig ./tsconfig.json --out ./data/schemas/options.json",
49
50
  "make-tarballs": "node --no-warnings ./dist/build-tools/MakeTarballsForInstalledPackages.js"
50
51
  },
@@ -52,8 +53,8 @@
52
53
  "echogarden": "./dist/cli/CLILauncher.js"
53
54
  },
54
55
  "dependencies": {
55
- "@aws-sdk/client-polly": "~3.1106.0",
56
- "@aws-sdk/client-transcribe-streaming": "~3.1106.0",
56
+ "@aws-sdk/client-polly": "~3.1127.0",
57
+ "@aws-sdk/client-transcribe-streaming": "~3.1127.0",
57
58
  "@echogarden/audio-io": "~0.4.2",
58
59
  "@echogarden/fasttext-wasm": "~0.1.0",
59
60
  "@echogarden/flite-wasi": "~0.1.1",
@@ -64,7 +65,7 @@
64
65
  "@echogarden/sonic-wasm": "~0.2.0",
65
66
  "@echogarden/speex-resampler-wasm": "~0.3.0",
66
67
  "@echogarden/svoxpico-wasm": "~0.2.0",
67
- "@echogarden/text-segmentation": "~0.7.0",
68
+ "@echogarden/text-segmentation": "~0.8.1",
68
69
  "@echogarden/transformers-nodejs-lite": "~2.17.1-lite.4",
69
70
  "@echogarden/wave-codec": "~0.4.0",
70
71
  "@echogarden/whisper.cpp-binding": "~0.2.2",
@@ -74,25 +75,25 @@
74
75
  "easier-http-request": "~0.1.0",
75
76
  "fs-extra": "~11.4.0",
76
77
  "graceful-fs": "~4.2.11",
77
- "html-to-text": "~10.0.0",
78
+ "html-to-text": "~10.0.1",
78
79
  "import-meta-resolve": "~4.2.0",
79
80
  "jieba-wasm": "~2.4.0",
80
81
  "jsdom": "~30.0.1",
81
82
  "kuromoji": "~0.1.2",
82
83
  "microsoft-cognitiveservices-speech-sdk": "~1.51.0",
83
- "msgpack-lite": "~0.1.26",
84
+ "msgpack-lite": "~0.2.2",
84
85
  "onnxruntime-node": "~1.21.1",
85
- "openai": "~7.4.0",
86
- "quick-json5": "~0.4.3",
87
- "regexp-composer": "~0.4.0",
86
+ "openai": "~7.10.0",
87
+ "quick-json5": "~0.5.0",
88
+ "regexp-composer": "~0.7.0",
88
89
  "sam-js": "~0.3.1",
89
90
  "standard-html-escaper": "~0.2.1",
90
91
  "strip-ansi": "~7.2.0",
91
92
  "tar": "~7.5.22",
92
93
  "tiktoken": "~1.0.22",
93
94
  "tinyld": "~1.3.4",
94
- "wasm-feature-detect": "~1.8.0",
95
- "wasm-heap-manager": "~0.4.0",
95
+ "wasm-feature-detect": "~1.9.0",
96
+ "wasm-heap-manager": "~0.5.1",
96
97
  "ws": "~8.21.3",
97
98
  "wtf_wikipedia": "~10.4.2"
98
99
  },
@@ -118,10 +119,11 @@
118
119
  "@types/graceful-fs": "~4.1.9",
119
120
  "@types/jsdom": "~30.0.0",
120
121
  "@types/msgpack-lite": "~0.1.12",
121
- "@types/node": "~26.2.0",
122
+ "@types/node": "~26.5.0",
122
123
  "@types/ws": "~8.18.1",
123
124
  "ts-json-schema-generator": "~2.9.0",
124
- "typescript": "~7.0.2"
125
+ "typescript": "~7.0.2",
126
+ "vitest": "~5.0.0"
125
127
  },
126
128
  "overrides": {
127
129
  "whatwg-url": "~14.0.0",