echogarden 2.0.14 → 2.1.1

package/README.md

@@ -11,7 +11,7 @@ Echogarden is an easy-to-use speech toolset that includes a variety of speech pr
 
 ## Features
 
-* **Text-to-speech** using the [VITS](https://github.com/jaywalnut310/vits) neural architecture, and [15 other offline and online engines](docs/Engines.md), including cloud services by [Google](https://cloud.google.com/text-to-speech), [Microsoft](https://azure.microsoft.com/en-us/products/ai-services/text-to-speech/), [Amazon](https://aws.amazon.com/polly/), [OpenAI](https://platform.openai.com/) and [Elevenlabs](https://elevenlabs.io/)
+* **Text-to-speech** using high-quality [Kokoro](https://github.com/hexgrad/kokoro) and [VITS](https://github.com/jaywalnut310/vits) offline voices, for many languages and dialects, and [16 other offline and online engines](docs/Engines.md), including cloud services by [Google](https://cloud.google.com/text-to-speech), [Microsoft](https://azure.microsoft.com/en-us/products/ai-services/text-to-speech/), [Amazon](https://aws.amazon.com/polly/), [OpenAI](https://platform.openai.com/) and [Elevenlabs](https://elevenlabs.io/)
 * **Speech-to-text** using a custom TypeScript/ONNX port of the [OpenAI Whisper](https://openai.com/research/whisper) speech recognition architecture, [whisper.cpp](https://github.com/ggerganov/whisper.cpp), and [several other engines](docs/Engines.md), including cloud services by [Google](https://cloud.google.com/speech-to-text), [Microsoft](https://azure.microsoft.com/en-us/products/ai-services/speech-to-text/), [Amazon](https://aws.amazon.com/transcribe/) and [OpenAI](https://platform.openai.com/)
 * **Speech-to-transcript alignment** using several variants of [dynamic time warping](https://en.wikipedia.org/wiki/Dynamic_time_warping) (DTW, DTW-RA), including support for multi-pass (hierarchical) processing, or via guided decoding using Whisper recognition models. Supports 100+ languages
 * **Speech-to-text translation**, translates speech in any of the [98 languages](https://platform.openai.com/docs/guides/speech-to-text/supported-languages) supported by Whisper, to English, with near word-level timing for the translated transcript

package/data/lexicons/heteronyms.en.json

@@ -167,6 +167,48 @@
 				"example": "There's no use."
 			}
 		],
+		"close": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "k l ˈoʊ z",
+						"en-gb-x-rp": "k l ˈoʊ z"
+					}
+				},
+
+				"pos": [
+					"VB"
+				],
+
+				"precededBy": [
+					"to", "we", "you", "they", "who", "i", "me", "will", "also", "don't", "didn't", "not", "doesn't", "please", "we'll", "would", "wouldn't", "should", "shouldn't", "must", "mustn't", "shall", "can", "can't", "may", "might", "can", "could", "couldn't"
+				],
+
+				"notPrecededBy": [
+					"the", "a", "be", "for", "no", "in", "of", "make", "its", "single", "their", "good", "making", "her", "his", "its", "our", "your", "my", "any", "personal"
+				],
+
+				"succeededBy": [
+					"the", "a", "an", "their", "it", "this", "them", "his", "her", "our", "your", "its", "these", "all", "my", "any", "those", "some", "one", "more", "every", "up", "down", "other"
+				],
+
+				"notSucceededBy": [
+					"to", "with"
+				],
+
+				"example": "I will close the door."
+			},
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "k l ˈoʊ s",
+						"en-gb-x-rp": "k l ˈoʊ s"
+					}
+				},
+
+				"example": "This seems very close."
+			}
+		],
 		"live": [
 			{
 				"pronunciation": {
@@ -226,7 +268,7 @@
 				],
 
 				"notPrecededBy": [
-					"the", "to", "their", "our", "these", "those", "some", "saved", "matter", "few", "some", "many", "innocent"
+					"the", "to", "their", "our", "these", "those", "some", "saved", "matter", "few", "some", "many", "innocent", "improve", "better"
 				],
 
 				"succeededBy": [
@@ -854,7 +896,7 @@
 				],
 
 				"succeededBy": [
-					"down", "up", "through", "the", "it", "each", "a", "an", "you", "me", "us", "them", "your", "their", "his", "her", "herself", "himself", "itself", "everybody", "our"
+					"down", "up", "out", "through", "the", "it", "each", "a", "an", "you", "me", "us", "them", "your", "their", "his", "her", "herself", "himself", "itself", "everybody", "our"
 				],
 
 				"notSucceededBy": [
@@ -1300,11 +1342,11 @@
 				],
 
 				"precededBy": [
-					"to", "i", "me", "us", "they", "we", "you", "will", "would", "can", "can't", "may", "might", "would", "wouldn't", "won't", "don't", "doesn't", "should", "shouldn't", "could", "couldn't", "must", "mustn't", "shall", "did", "didn't", "then"
+					"to", "i", "me", "us", "they", "we", "you", "will", "would", "can", "can't", "may", "might", "would", "wouldn't", "won't", "don't", "doesn't", "should", "shouldn't", "could", "couldn't", "must", "mustn't", "shall", "did", "didn't", "then", "people"
 				],
 
 				"notPrecededBy": [
-					"of", "for", "this", "similar", "with", "criminal", "dangerous", "ethical", "unethical", "good", "future", "disorderly"
+					"of", "for", "this", "similar", "with", "criminal", "dangerous", "ethical", "unethical", "good", "future", "disorderly", "his", "her", "our", "my"
 				],
 
 				"succeededBy": [
@@ -1629,7 +1671,7 @@
 				],
 
 				"precededBy": [
-					"the", "a", "all", "his", "her", "its", "our", "be", "very"
+					"very", "quite"
 				],
 
 				"notPrecededBy": [

package/data/lexicons/words.en.json

@@ -0,0 +1,184 @@
+{
+	"en": {
+		"vs.": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "v ˈɜː s ᵻ z",
+						"en-gb-x-rp": "v ˈɜː s ɪ z"
+					}
+				}
+			}
+		],
+		"inherently": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "ɪ n h ˈɛ ɹ ə n t l i",
+						"en-gb-x-rp": "ɪ n h ˈɛ ɹ ə n t l i"
+					}
+				}
+			}
+		],
+		"subsection": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "s ˈʌ b s ɛ k ʃ ə n",
+						"en-gb-x-rp": "s ˈʌ b s ɛ k ʃ ə n"
+					}
+				}
+			}
+		],
+		"subsections": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "s ˈʌ b s ɛ k ʃ ə n z",
+						"en-gb-x-rp": "s ˈʌ b s ɛ k ʃ ə n z"
+					}
+				}
+			}
+		],
+		"subvector": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "s ˈə b v ɛ k t ɚ",
+						"en-gb-x-rp": "s ˈə b v ɛ k t ɐ"
+					}
+				}
+			}
+		],
+		"subvectors": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "s ˈə b v ɛ k t ɚ z",
+						"en-gb-x-rp": "s ˈə b v ɛ k t ɐ z"
+					}
+				}
+			}
+		],
+		"recommender": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "ɹ ˌiː k ə m ˈɛ n d ɚ",
+						"en-gb-x-rp": "ɹ ˌiː k ə m ˈɛ n d ɐ"
+					}
+				}
+			}
+		],
+		"representation": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "ɹ ˌɛ p ɹ ə z ɛ n t ˈeɪ ʃ ə n",
+						"en-gb-x-rp": "ɹ ˌɛ p ɹ ᵻ z ɛ n t ˈeɪ ʃ ə n"
+					}
+				}
+			}
+		],
+		"representations": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "ɹ ˌɛ p ɹ ə z ɛ n t ˈeɪ ʃ ə n z",
+						"en-gb-x-rp": "ɹ ˌɛ p ɹ ᵻ z ɛ n t ˈeɪ ʃ ə n z"
+					}
+				}
+			}
+		],
+		"residual": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "ɹ ə z ˈɪ dʒ uː əl",
+						"en-gb-x-rp": "ɹ ᵻ z ˈɪ dʒ ʊ l"
+					}
+				}
+			}
+		],
+		"residuals": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "ɹ ə z ˈɪ dʒ uː əl z",
+						"en-gb-x-rp": "ɹ ᵻ z ˈɪ dʒ ʊ l z"
+					}
+				}
+			}
+		],
+		"kokoro": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "k ˈoʊ k ə ɹ oʊ",
+						"en-gb-x-rp": "k ˈəʊ k ə ɹ əʊ"
+					}
+				}
+			}
+		],
+		"query": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "k w ˈɛ ɹ i",
+						"en-gb-x-rp": "k w ˈiə ɹ i"
+					}
+				}
+			}
+		],
+		"queries": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "k w ˈɛ ɹ i z",
+						"en-gb-x-rp": "k w ˈiə ɹ i z"
+					}
+				}
+			}
+		],
+		"jupyter": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "dʒ ˈuː p ɪ ɾ ɚ",
+						"en-gb-x-rp": "dʒ ˈuː p ɪ t ɐ"
+					}
+				}
+			}
+		],
+		"movies": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "m ˈuː v i z",
+						"en-gb-x-rp": "m ˈuː v i z"
+					}
+				}
+			}
+		],
+		"preprocessing": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "p ɹ ˈi p ɹ ˌɑː s ɛ s ɪ ŋ",
+						"en-gb-x-rp": "p ɹ ˈi p ɹ ˌəʊ s ɛ s ɪ ŋ"
+					}
+				}
+			}
+		],
+		"annoy": [
+			{
+				"pronunciation": {
+					"espeak": {
+						"en-us": "ə n ˈɔ ɪ",
+						"en-gb-x-rp": "ə n ˈɔ ɪ"
+					}
+				}
+			}
+		]
+	}
+}

package/data/schemas/options.json

@@ -191,6 +191,22 @@
           },
           "additionalProperties": false
         },
+        "kokoro": {
+          "type": "object",
+          "properties": {
+            "provider": {
+              "$ref": "#/definitions/OnnxExecutionProvider"
+            },
+            "model": {
+              "type": "string",
+              "enum": [
+                "82m-v1.0-fp32",
+                "82m-v1.0-quantized"
+              ]
+            }
+          },
+          "additionalProperties": false
+        },
         "pico": {
           "type": "object",
           "additionalProperties": false
@@ -199,6 +215,21 @@
           "type": "object",
           "additionalProperties": false
         },
+        "gnuspeech": {
+          "type": "object",
+          "properties": {
+            "tempo": {
+              "type": "number"
+            },
+            "controlRate": {
+              "type": "number"
+            },
+            "debug": {
+              "type": "boolean"
+            }
+          },
+          "additionalProperties": false
+        },
         "espeak": {
           "type": "object",
           "properties": {
@@ -376,8 +407,10 @@
       "type": "string",
       "enum": [
         "vits",
+        "kokoro",
         "pico",
         "flite",
+        "gnuspeech",
         "espeak",
         "sam",
         "sapi",
@@ -1471,6 +1504,22 @@
           },
           "additionalProperties": false
         },
+        "kokoro": {
+          "type": "object",
+          "properties": {
+            "provider": {
+              "$ref": "#/definitions/OnnxExecutionProvider"
+            },
+            "model": {
+              "type": "string",
+              "enum": [
+                "82m-v1.0-fp32",
+                "82m-v1.0-quantized"
+              ]
+            }
+          },
+          "additionalProperties": false
+        },
         "pico": {
           "type": "object",
           "additionalProperties": false
@@ -1479,6 +1528,21 @@
           "type": "object",
           "additionalProperties": false
         },
+        "gnuspeech": {
+          "type": "object",
+          "properties": {
+            "tempo": {
+              "type": "number"
+            },
+            "controlRate": {
+              "type": "number"
+            },
+            "debug": {
+              "type": "boolean"
+            }
+          },
+          "additionalProperties": false
+        },
         "espeak": {
           "type": "object",
           "properties": {

package/dist/api/Synthesis.d.ts

@@ -14,7 +14,7 @@ export interface SynthesisResult {
     language: string;
     voice: string;
 }
-export type SynthesisEngine = 'vits' | 'pico' | 'flite' | 'espeak' | 'sam' | 'sapi' | 'msspeech' | 'coqui-server' | 'google-cloud' | 'microsoft-azure' | 'amazon-polly' | 'openai-cloud' | 'elevenlabs' | 'google-translate' | 'microsoft-edge' | 'streamlabs-polly';
+export type SynthesisEngine = 'vits' | 'kokoro' | 'pico' | 'flite' | 'gnuspeech' | 'espeak' | 'sam' | 'sapi' | 'msspeech' | 'coqui-server' | 'google-cloud' | 'microsoft-azure' | 'amazon-polly' | 'openai-cloud' | 'elevenlabs' | 'google-translate' | 'microsoft-edge' | 'streamlabs-polly';
 export type TimePitchShiftingMethod = 'sonic' | 'rubberband';
 export interface SynthesisOptions {
     engine?: SynthesisEngine;
@@ -50,8 +50,17 @@ export interface SynthesisOptions {
         speakerId?: number;
         provider?: OnnxExecutionProvider;
     };
+    kokoro?: {
+        provider?: OnnxExecutionProvider;
+        model?: '82m-v1.0-fp32' | '82m-v1.0-quantized';
+    };
     pico?: {};
     flite?: {};
+    gnuspeech?: {
+        tempo?: number;
+        controlRate?: number;
+        debug?: boolean;
+    };
     espeak?: {
         rate?: number;
         pitch?: number;

package/dist/api/Synthesis.js

@@ -110,9 +110,9 @@ async function synthesizeSegments(segments, options, onSegment, onSentence) {
             const sentenceText = sentences[sentenceIndex].trim();
             logger.log(`\n${chalk.magentaBright(`Synthesizing sentence ${sentenceIndex + 1}/${sentences.length}`)}: "${sentenceText}"`);
             const sentenceStartTime = timeOffset;
-            let sentencetSynthesisOptions = { postProcessing: { normalizeAudio: false } };
-            sentencetSynthesisOptions = extendDeep(options, sentencetSynthesisOptions);
-            const { synthesizedAudio: sentenceRawAudio, timeline: sentenceTimeline } = await synthesizeSegment(sentenceText, sentencetSynthesisOptions);
+            let sentenceSynthesisOptions = { postProcessing: { normalizeAudio: false } };
+            sentenceSynthesisOptions = extendDeep(options, sentenceSynthesisOptions);
+            const { synthesizedAudio: sentenceRawAudio, timeline: sentenceTimeline } = await synthesizeSegment(sentenceText, sentenceSynthesisOptions);
             const endPause = sentenceIndex == sentences.length - 1 ? options.segmentEndPause : options.sentenceEndPause;
             sentenceRawAudio.audioChannels[0] = trimAudioEnd(sentenceRawAudio.audioChannels[0], endPause * sentenceRawAudio.sampleRate);
             sentencesRawAudio.push(sentenceRawAudio);
@@ -284,6 +284,27 @@ async function synthesizeSegment(text, options) {
             logger.end();
             break;
         }
+        case 'kokoro': {
+            if (inputIsSSML) {
+                throw new Error(`The Kokoro engine doesn't currently support SSML inputs`);
+            }
+            const kokoroOptions = options.kokoro;
+            const kokoroTTS = await import('../synthesis/KokoroTTS.js');
+            const lexicons = await loadLexiconsForLanguage(language, options.customLexiconPaths);
+            const onnxExecutionProviders = kokoroOptions.provider ? [kokoroOptions.provider] : [];
+            const modelName = kokoroOptions.model;
+            const modelPackageName = `kokoro-${modelName}`;
+            const modelPath = await loadPackage(modelPackageName);
+            const voicesPath = await loadPackage('kokoro-82m-v1.0-voices');
+            logger.end();
+            logger.logTitledMessage(`Using model`, modelPackageName);
+            const { rawAudio, timeline: outTimeline } = await kokoroTTS.synthesizeSentence(text, selectedVoice, speed, lexicons, modelPath, voicesPath, onnxExecutionProviders);
+            synthesizedAudio = rawAudio;
+            timeline = outTimeline;
+            shouldPostprocessPitch = true;
+            logger.end();
+            break;
+        }
         case 'pico': {
             if (inputIsSSML) {
                 throw new Error(`The SVOX Pico engine doesn't currently support SSML inputs`);
@@ -312,6 +333,24 @@ async function synthesizeSegment(text, options) {
             shouldPostprocessPitch = true;
             break;
         }
+        case 'gnuspeech': {
+            if (inputIsSSML) {
+                throw new Error(`The Gnuspeech engine doesn't currently support SSML inputs`);
+            }
+            const engineOptions = options.gnuspeech;
+            const GnuSpeech = await import('../synthesis/GnuSpeechTTS.js');
+            const { defaultGnuSpeechOptions } = await import('@echogarden/gnuspeech-wasm');
+            const gnuSpeechOptions = extendDeep(defaultGnuSpeechOptions, engineOptions);
+            if (!engineOptions.tempo) {
+                gnuSpeechOptions.tempo = speed;
+            }
+            await logger.startAsync(`Synthesize with Gnuspeech`);
+            const { rawAudio } = await GnuSpeech.synthesize(simplifiedText, gnuSpeechOptions);
+            synthesizedAudio = rawAudio;
+            shouldPostprocessPitch = true;
+            logger.end();
+            break;
+        }
         case 'espeak': {
             const EspeakTTS = await import('../synthesis/EspeakTTS.js');
             const engineOptions = options.espeak;
@@ -652,7 +691,7 @@ async function synthesizeSegment(text, options) {
         timeline = timeline.filter(entry => isWordOrSymbolWord(entry.text));
     }
     logger.end();
-    logger.logDuration('Segment synthesis time', startTimestamp, chalk.magentaBright);
+    logger.logDuration('Part synthesis time', startTimestamp, chalk.magentaBright);
     return { synthesizedAudio, timeline };
 }
 function convertSpeedScaleToSSMLValueString(rate) {
@@ -728,13 +767,19 @@ export const defaultSynthesisOptions = {
         speakerId: undefined,
         provider: undefined,
     },
+    kokoro: {
+        model: '82m-v1.0-fp32'
+    },
     pico: {},
     flite: {},
+    gnuspeech: {
+        debug: false,
+    },
     espeak: {
         rate: undefined,
         pitch: undefined,
         pitchRange: undefined,
-        useKlatt: false
+        useKlatt: false,
     },
     sam: {
         speed: undefined,
@@ -826,6 +871,11 @@ export async function requestVoiceList(options) {
                 voiceList = SvoxPicoTTS.voiceList;
                 break;
             }
+            case 'gnuspeech': {
+                const GnuSpeech = await import('../synthesis/GnuSpeechTTS.js');
+                voiceList = GnuSpeech.voiceList;
+                break;
+            }
             case 'sam': {
                 voiceList.push({
                     name: 'sam',
@@ -841,6 +891,11 @@ export async function requestVoiceList(options) {
                 });
                 break;
             }
+            case 'kokoro': {
+                const KokoroTTS = await import('../synthesis/KokoroTTS.js');
+                voiceList = KokoroTTS.voiceList;
+                break;
+            }
             case 'sapi': {
                 const SapiTTS = await import('../synthesis/SapiTTS.js');
                 await SapiTTS.AssertSAPIAvailable(false);
@@ -1066,28 +1121,40 @@ export const synthesisEngines = [
         description: 'A high-quality end-to-end neural speech synthesis architecture.',
         type: 'local'
     },
+    {
+        id: 'kokoro',
+        name: 'Kokoro',
+        description: 'A high-quality neural speech synthesis model based on the StyleTTS 2 architecture.',
+        type: 'local'
+    },
     {
         id: 'pico',
         name: 'SVOX Pico',
-        description: 'A legacy diphone-based synthesis engine.',
+        description: 'A legacy diphone-based speech synthesizer.',
         type: 'local'
     },
     {
         id: 'flite',
         name: 'Flite',
-        description: 'A legacy diphone-based synthesis engine.',
+        description: 'A legacy diphone-based speech synthesizer.',
+        type: 'local'
+    },
+    {
+        id: 'gnuspeech',
+        name: 'Gnuspeech',
+        description: 'A legacy articulatory speech synthesizer.',
         type: 'local'
     },
     {
         id: 'espeak',
         name: 'eSpeak NG',
-        description: `A lightweight 'robot' sounding formant-based synthesizer.`,
+        description: `A lightweight, highly multilingual, 'robot'-like formant-based speech synthesizer.`,
         type: 'local'
     },
     {
         id: 'sam',
         name: 'SAM (Software Automatic Mouth)',
-        description: `A classic 'robot' speech synthesizer from 1982.`,
+        description: `A classic 'robot'-like speech synthesizer from 1982.`,
         type: 'local'
     },
     {