echogarden 2.1.1 → 2.2.0

package/docs/Technical.md

@@ -2,36 +2,43 @@
 
 * Echogarden is written in TypeScript and targets the Node.js platform.
 * It uses ESM modules and latest ECMAScript and TypeScript features.
-* It does not depend on essential binary executables. Instead, all of its engines either use pure JavaScript, WebAssembly, WASI, or the ONNX runtime, with some exceptions: the CLI does invoke a command line `ffmpeg` tool, auto-downloaded from internal packages. Using expansion packages simplifies the installation and ensures non-buggy version are used. Since SoX `v14.4.2` is broken on Windows, it bundles `v14.4.1`.
+* It does not depend on essential binary executables. Instead, all of its engines either use pure JavaScript, WebAssembly, WASI, or the ONNX runtime, with some exceptions: the CLI does invoke a command line `ffmpeg` tool, auto-downloaded using its internal package system.
 * It does not depend on essential native node.js modules requiring install-time compilation with `node-gyp`. This greatly simplifies the installation experience for end-users (the ONNX runtime bundles precompiled NAPI modules for all supported platforms - it doesn't require any compilation during its installation).
 
 ## Package system
 
 Echogarden uses its own package system to download and install various components as needed. These components can be TTS voices, STT models, or other types of models and data.
 
-Packages are downloaded as `.tar.gz` files, and are extracted to `[data-folder]/packages/[package-name]`. Each package has its own subdirectory.
+Packages are downloaded as `.tar.gz` files, and are extracted to `[data-folder]/packages/[package-id-string]`. Each package has its own subdirectory.
 
 `[data-folder]` is located at:
 * `%AppData%\Local\echogarden` on Windows
 * `Users/User/Library/Application Support/echogarden` on macOS
 * `/home/user/.local/share/echogarden` on Linux
 
+`[package-id-string]` is formatted as:
+```
+[engine name]-[package id]-[date as yyyymmdd]
+```
+
 By using downloadable packages, the installed size is made significantly smaller and the installation faster. The total size of all available packages is currently about 50GB (may not be up-to-date).
 
-The packages are currently hosted and downloaded from a dedicated [Hugging Face repository](https://huggingface.co/echogarden/echogarden-packages).
+The packages are currently hosted and downloaded from a [dedicated Hugging Face repository](https://huggingface.co/echogarden/echogarden-packages).
 
 ## Can the base install size be made smaller?
 
-The base installed (uncompressed) size, including dependencies, is around 270MB. This is without any models or voices, which are downloaded as needed.
+The base installed (uncompressed) size, including dependencies, is around 400MB (may not be up-to-date). This is without any models or voices, which are downloaded as needed.
 
 Currently, the largest contributors to the size are:
 
-* `onnxruntime-node` (NAPI): 133MB
-* `kuromoji` (JavaScript) 40MB
-* `flite-wasi` (WASI): 20MB
-* `espeak-ng-emscripten` (WASM): 18MB
+* `onnxruntime-node` (core inference runtime, NAPI): 180MB
+* `kuromoji` (Japanese tokenizer, JavaScript) 40MB
+* `espeak-ng-emscripten` (core synthesis and phonemization engine, WASM): 23MB
+* `tiktoken` (tokenizer library, WASM): 22MB
+* `flite-wasi` (synthesis engine, WASI): 20MB
+* `jieba-wasm` (Chinese tokenizer, WASM): 14MB
 
-`onnxruntime-node` is large because it bundles pre-compiled binaries for multiple platforms. `kuromoji` is large because of its dictionary files and some unessential test code it bundles. The other three packages include large WASM binaries.
+`onnxruntime-node` is large because it bundles pre-compiled binaries for multiple platforms. `kuromoji` is large because of its dictionary files and some unessential test code it bundles. The other packages include large WASM binaries.
 
 So, yes, in the future it may be possible to reduce the core installed size by dynamically installing some of these dependencies, or using modified, "slimmed-down" versions of some packages.
 
@@ -44,7 +51,7 @@ However, it is a lot of work, and only a subset of the engines can be supported
 * Significantly slower inference when using CPU for ONNX models
 * No cross-domain network connectivity - can't connect to Google Cloud, Microsoft, Amazon etc. without a proxy
 * Large initial download size would make it too heavy and slow to load as part of a standard web page directly
-* Large memory requirement for the VITS models, starting at about 800MB - 1GB, which is a bit too much for a browser
+* Large memory requirements for the various ONNX models, starting at about 800MB - 1GB, which is a bit too much for a browser
 * Due to the high code complexity, data size, and memory consumption, it is unlikely that a browser extension, internally bundling some of the models, would be accepted to the Chrome and Firefox web stores
 * Will require a virtual file system to store models and make use of downloadable packages
 * Requires duplicating a lot of prior work, porting many node.js-only APIs, and increasing code complexity
@@ -71,5 +78,5 @@ Also, `option=value` is more similar to the syntax used in the configuration fil
 
 * `src`: TypeScript source code
 * `dist`: compiled JavaScript modules
-* `data`: various data files, including phonetic lexicons and language code conversion tables. `data/schemas` stores JSON schemas for all configuration options, auto-generated using [`ts-json-schema-generator`](https://github.com/vega/ts-json-schema-generator) directly from the TypeScript code, and used by the CLI to parse and validate the options provided
+* `data`: various data files, including phonetic lexicons and language code conversion tables. `data/schemas` stores JSON schemas for all configuration options, auto-generated using [`ts-json-schema-generator`](https://github.com/vega/ts-json-schema-generator) directly from the TypeScript code, and used by the CLI to parse and validate the options provided. `data/lexicons` contains phonetic lexicons used in speech synthesis and alignment
 * `docs`: documentation

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "2.1.1",
+	"version": "2.2.0",
 	"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -52,8 +52,8 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "^3.747.0",
-		"@aws-sdk/client-transcribe-streaming": "^3.744.0",
+		"@aws-sdk/client-polly": "^3.750.0",
+		"@aws-sdk/client-transcribe-streaming": "^3.750.0",
 		"@echogarden/audio-io": "^0.3.0",
 		"@echogarden/espeak-ng-emscripten": "^0.3.3",
 		"@echogarden/fasttext-wasm": "^0.1.0",
@@ -85,14 +85,14 @@
 		"microsoft-cognitiveservices-speech-sdk": "^1.42.0",
 		"msgpack-lite": "^0.1.26",
 		"onnxruntime-node": "^1.20.1",
-		"openai": "^4.84.0",
+		"openai": "^4.85.3",
 		"sam-js": "^0.3.1",
 		"strip-ansi": "^7.1.0",
 		"tar": "^7.4.3",
 		"tiktoken": "^1.0.20",
 		"tinyld": "^1.3.4",
 		"wasm-feature-detect": "^1.8.0",
-		"ws": "^8.18.0",
+		"ws": "^8.18.1",
 		"wtf_wikipedia": "^10.4.0"
 	},
 	"peerDependencies": {
@@ -112,7 +112,7 @@
 		"@types/graceful-fs": "^4.1.9",
 		"@types/jsdom": "^21.1.7",
 		"@types/msgpack-lite": "^0.1.11",
-		"@types/node": "^22.13.2",
+		"@types/node": "^22.13.5",
 		"@types/tar": "^6.1.13",
 		"@types/ws": "^8.5.14",
 		"ts-json-schema-generator": "^2.3.0",

package/src/api/Recognition.ts

@@ -15,6 +15,7 @@ import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 import { type OpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
 import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
 import { type SileroRecognitionOptions } from '../recognition/SileroSTT.js'
+import { type DeepgramSTTOptions } from '../recognition/DeepgramSTT.js'
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 
 const log = logToStderr
@@ -274,6 +275,22 @@ export async function recognize(input: AudioSourceParam, options: RecognitionOpt
 			break
 		}
 
+		case 'deepgram': {
+			const DeepgramSTT = await import('../recognition/DeepgramSTT.js')
+
+			const deepgramOptions = options.deepgram!
+
+			if (!deepgramOptions.apiKey) {
+				throw new Error(`No Deepgram API key provided`)
+			}
+
+			logger.end();
+
+			({ transcript, timeline } = await DeepgramSTT.recognize(sourceRawAudio, options.language ? shortLanguageCode : undefined, deepgramOptions))
+
+			break
+		}
+
 		default: {
 			throw new Error(`Engine '${options.engine}' is not supported`)
 		}
@@ -330,7 +347,7 @@ export interface RecognitionResult {
 	backgroundRawAudio?: RawAudio
 }
 
-export type RecognitionEngine = 'whisper' | 'whisper.cpp' | 'vosk' | 'silero' | 'google-cloud' | 'microsoft-azure' | 'amazon-transcribe' | 'openai-cloud'
+export type RecognitionEngine = 'whisper' | 'whisper.cpp' | 'vosk' | 'silero' | 'google-cloud' | 'microsoft-azure' | 'amazon-transcribe' | 'openai-cloud' | 'deepgram'
 
 export interface RecognitionOptions {
 	engine?: RecognitionEngine
@@ -383,6 +400,8 @@ export interface RecognitionOptions {
 	}
 
 	openAICloud?: OpenAICloudSTTOptions
+
+	deepgram?: DeepgramSTTOptions
 }
 
 export const defaultRecognitionOptions: RecognitionOptions = {
@@ -443,6 +462,9 @@ export const defaultRecognitionOptions: RecognitionOptions = {
 
 	openAICloud: {
 	},
+
+	deepgram: {
+	}
 }
 
 export const recognitionEngines: API.EngineMetadata[] = [
@@ -494,4 +516,10 @@ export const recognitionEngines: API.EngineMetadata[] = [
 		description: 'OpenAI cloud speech-to-text service.',
 		type: 'cloud'
 	},
+	{
+		id: 'deepgram',
+		name: 'Deepgram',
+		description: 'Deepgram cloud speech-to-text service.',
+		type: 'cloud'
+	},
 ]

package/src/api/Synthesis.ts

@@ -20,8 +20,8 @@ import { shouldCancelCurrentTask } from '../server/Worker.js'
 import chalk from 'chalk'
 import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 import { type EspeakOptions } from '../synthesis/EspeakTTS.js'
-import { type OpenAICloudTTSOptions  } from '../synthesis/OpenAICloudTTS.js'
-import { type ElevenlabsTTSOptions } from '../synthesis/ElevenlabsTTS.js'
+import { type OpenAICloudTTSOptions } from '../synthesis/OpenAICloudTTS.js'
+import { type ElevenLabsTTSOptions } from '../synthesis/ElevenLabsTTS.js'
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 import { simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
 import { convertHtmlToText } from '../utilities/StringUtilities.js'
@@ -63,7 +63,7 @@ async function synthesizeSegments(segments: string[], options: SynthesisOptions,
 		let segmentsPlainText = segments
 
 		if (options.ssml) {
-			segmentsPlainText= []
+			segmentsPlainText = []
 
 			for (const segment of segments) {
 				segmentsPlainText.push(await convertHtmlToText(segment))
@@ -811,7 +811,7 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 				throw new Error(`The Elevenlabs engine doesn't support SSML inputs`)
 			}
 
-			const ElevenLabsTTS = await import('../synthesis/ElevenlabsTTS.js')
+			const ElevenLabsTTS = await import('../synthesis/ElevenLabsTTS.js')
 
 			const engineOptions = options.elevenlabs!
 
@@ -1198,7 +1198,7 @@ export interface SynthesisOptions {
 
 	openAICloud?: OpenAICloudTTSOptions
 
-	elevenlabs?: ElevenlabsTTSOptions,
+	elevenlabs?: ElevenLabsTTSOptions,
 
 	googleTranslate?: {
 		tld?: string
@@ -1593,14 +1593,14 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 			}
 
 			case 'elevenlabs': {
-				const ElevenLabsTTS = await import('../synthesis/ElevenlabsTTS.js')
+				const ElevenLabsTTS = await import('../synthesis/ElevenLabsTTS.js')
 
 				const engineOptions = options.elevenlabs!
 
 				const apiKey = engineOptions.apiKey
 
 				if (!apiKey) {
-					throw new Error(`No Elevenlabs API key provided`)
+					throw new Error(`No ElevenLabs API key provided`)
 				}
 
 				voiceList = await ElevenLabsTTS.getVoiceList(apiKey)

package/src/codecs/FFMpegTranscoder.ts

@@ -223,6 +223,7 @@ export function getDefaultFFMpegOptionsForSpeech(fileExtension: string, customBi
 		}
 	} else if (fileExtension == 'opus') {
 		ffmpegOptions = {
+			format: 'ogg',
 			codec: 'libopus',
 			bitrate: 48,
 			customOptions: []
@@ -236,6 +237,7 @@ export function getDefaultFFMpegOptionsForSpeech(fileExtension: string, customBi
 		}
 	} else if (fileExtension == 'ogg') {
 		ffmpegOptions = {
+			format: 'ogg',
 			codec: 'libvorbis',
 			bitrate: 48,
 			customOptions: []

package/src/nlp/EspeakPhonemizer.ts

@@ -7,14 +7,14 @@ const log = logToStderr
 export async function phonemizeSentence(sentence: string, espeakVoice: string, substitutionMap?: Map<string, string[]>, useIpa = true) {
 	const ipaString = await EspeakTTS.textToPhonemes(sentence, espeakVoice, useIpa)
 
-	const clauseStrings = ipaString.split(' | ')
+	const phraseStrings = ipaString.split(' | ')
 
-	const clauses: string[][][] = []
+	const phrases: string[][][] = []
 
-	for (let clauseIndex = 0; clauseIndex < clauseStrings.length; clauseIndex++) {
-		const clauseString = clauseStrings[clauseIndex]
+	for (let phraseIndex = 0; phraseIndex < phraseStrings.length; phraseIndex++) {
+		const phraseString = phraseStrings[phraseIndex]
 
-		const wordStrings = clauseString.trim().split(/ +/g)
+		const wordStrings = phraseString.trim().split(/ +/g)
 		const words: string[][] = []
 
 		for (let wordIndex = 0; wordIndex < wordStrings.length; wordIndex++) {
@@ -40,11 +40,11 @@ export async function phonemizeSentence(sentence: string, espeakVoice: string, s
 		}
 
 		if (words.length > 0) {
-			clauses.push(words)
+			phrases.push(words)
 		}
 	}
 
-	return clauses
+	return phrases
 }
 
 export async function phonemizeText(text: string, voice: string, substitutionMap?: Map<string, string[]>) {
@@ -58,46 +58,46 @@ export async function phonemizeText(text: string, voice: string, substitutionMap
 			.replaceAll('»', ', ')
 
 	const segmentedText = await Segmentation.parse(text, voice)
-	const preparedClauses: string[] = []
-	const clauseBreakers: string[] = []
+	const preparedPhrases: string[] = []
+	const phraseBreakers: string[] = []
 
 	for (const sentence of segmentedText) {
-		for (const clause of sentence.phrases) {
-			const words = clause.words.filter(wordObject => Segmentation.isWordOrSymbolWord(wordObject.text))
-			const preparedClauseText = words.map(word => word.text.replace(/\./g, ' ')).join(' ')
+		for (const phrase of sentence.phrases) {
+			const words = phrase.words.filter(wordObject => Segmentation.isWordOrSymbolWord(wordObject.text))
+			const preparedPhraseText = words.map(word => word.text.replace(/\./g, ' ')).join(' ')
 
-			preparedClauses.push(preparedClauseText)
+			preparedPhrases.push(preparedPhraseText)
 
-			const trimmedClauseText = clause.text.trim()
-			const lastChar = trimmedClauseText[trimmedClauseText.length - 1]
+			const trimmedPhraseText = phrase.text.trim()
+			const lastChar = trimmedPhraseText[trimmedPhraseText.length - 1]
 
-			if (clause.isSentenceFinalizer) {
-				if (trimmedClauseText.endsWith('?') || trimmedClauseText.endsWith(`?"`)) {
-					clauseBreakers.push('?')
-				} else if (trimmedClauseText.endsWith('!') || trimmedClauseText.endsWith(`!"`)) {
-					clauseBreakers.push('!')
+			if (phrase.isSentenceFinalizer) {
+				if (trimmedPhraseText.endsWith('?') || trimmedPhraseText.endsWith(`?"`)) {
+					phraseBreakers.push('?')
+				} else if (trimmedPhraseText.endsWith('!') || trimmedPhraseText.endsWith(`!"`)) {
+					phraseBreakers.push('!')
 				} else {
-					clauseBreakers.push('.')
+					phraseBreakers.push('.')
 				}
 			} else {
 				if (lastChar == ':' || lastChar == ';') {
-					clauseBreakers.push(lastChar)
+					phraseBreakers.push(lastChar)
 				} else {
-					clauseBreakers.push(',')
+					phraseBreakers.push(',')
 				}
 			}
 		}
 	}
 
-	return phonemizeClauses(preparedClauses, voice, clauseBreakers, substitutionMap)
+	return phonemizePhrases(preparedPhrases, voice, phraseBreakers, substitutionMap)
 }
 
-export async function phonemizeClauses(clauses: string[], voice: string, clauseBreakers: string[], substitutionMap?: Map<string, string[]>) {
-	if (clauses.length == 0) {
+export async function phonemizePhrases(phrases: string[], voice: string, phraseBreakers: string[], substitutionMap?: Map<string, string[]>) {
+	if (phrases.length == 0) {
 		return []
 	}
 
-	const preparedText = clauses.join('\n\n') // filter(clause => clause.trim().length > 0)
+	const preparedText = phrases.join('\n\n') // filter(phrase => phrase.trim().length > 0)
 
 	const ipaString = await EspeakTTS.textToIPA(preparedText, voice)
 
@@ -130,31 +130,31 @@ export async function phonemizeClauses(clauses: string[], voice: string, clauseB
 		})
 	})
 
-	if (ipaLines.length != clauseBreakers.length) {
-		log(clauses)
+	if (ipaLines.length != phraseBreakers.length) {
+		log(phrases)
 		log(ipaLines)
-		log(clauseBreakers)
+		log(phraseBreakers)
 
-		throw new Error(`Unexpected: IPA lines count (${ipaLines.length}) is not equal to clause breakers count (${clauseBreakers.length})`)
+		throw new Error(`Unexpected: IPA lines count (${ipaLines.length}) is not equal to phrase breakers count (${phraseBreakers.length})`)
 	}
 
 	for (let i = 0; i < phonemeLines.length; i++) {
 		const line = phonemeLines[i]
 		const lastWordInLine = line[line.length - 1]
 
-		lastWordInLine.push(clauseBreakers[i])
+		lastWordInLine.push(phraseBreakers[i])
 	}
 
 	return phonemeLines
 }
 
-export function phonemizedClausesToSentences(phonemizedClauses: string[][][]) {
+export function phonemizedPhrasesToSentences(phonemizedPhrases: string[][][]) {
 	let phonemizedSentences: string[][][] = [[]]
 
-	for (const phonemizedClause of phonemizedClauses) {
-		phonemizedSentences[phonemizedSentences.length - 1].push(...phonemizedClause)
+	for (const phonemizedPhrase of phonemizedPhrases) {
+		phonemizedSentences[phonemizedSentences.length - 1].push(...phonemizedPhrase)
 
-		const lastWord = phonemizedClause[phonemizedClause.length - 1]
+		const lastWord = phonemizedPhrase[phonemizedPhrase.length - 1]
 		const lastPhoneme = lastWord[lastWord.length - 1]
 
 		if (['.', '?', '!'].includes(lastPhoneme)) {

package/src/nlp/Lexicon.ts

@@ -3,9 +3,9 @@ import { getShortLanguageCode } from '../utilities/Locale.js'
 import { resolveToModuleRootDir } from '../utilities/PathUtilities.js'
 
 export function tryGetFirstLexiconSubstitution(sentenceWords: string[], wordIndex: number, lexicons: Lexicon[], languageCode: string) {
-	const reversedLexicons = [...lexicons].reverse() // Give precedence to later lexicons
+	for (let i = lexicons.length - 1; i >= 0; i--) {
+		const lexicon = lexicons[i]
 
-	for (const lexicon of reversedLexicons) {
 		const match = tryGetLexiconSubstitution(sentenceWords, wordIndex, lexicon, languageCode)
 
 		if (match) {
@@ -20,51 +20,51 @@ export function tryGetLexiconSubstitution(sentenceWords: string[], wordIndex: nu
 	let word = sentenceWords[wordIndex]
 
 	if (!word) {
-		return
+		return undefined
 	}
 
 	const shortLanguageCode = getShortLanguageCode(languageCode)
 	const lexiconForLanguage = lexicon[shortLanguageCode]
 
 	if (!lexiconForLanguage) {
-		return
+		return undefined
 	}
 
 	const lexiconEntry = lexiconForLanguage[word]
 
 	if (!lexiconEntry) {
-		return
+		return undefined
 	}
 
 	for (let i = 0; i < lexiconEntry.length; i++) {
-		const substitutionEntry = lexiconEntry[i]
+		const candidateEntry = lexiconEntry[i]
 
-		const substitutionPhonemesText = substitutionEntry?.pronunciation?.espeak?.[languageCode]
+		const pronunciationPhonemesText = candidateEntry?.pronunciation?.espeak?.[languageCode]
 
-		if (!substitutionPhonemesText) {
+		if (!pronunciationPhonemesText) {
 			continue
 		}
 
-		const precedingWord = sentenceWords[wordIndex - 1] || ''
-		const succeedingWord = sentenceWords[wordIndex + 1] || ''
+		const precedingWord = sentenceWords[wordIndex - 1] ?? ''
+		const followingWord = sentenceWords[wordIndex + 1] ?? ''
 
-		const precededBy = substitutionEntry?.precededBy || []
-		const notPrecededBy = substitutionEntry?.notPrecededBy || []
+		const precededBy = candidateEntry?.precededBy ?? []
+		const notPrecededBy = candidateEntry?.notPrecededBy ?? []
 
-		const succeededBy = substitutionEntry?.succeededBy || []
-		const notSucceededBy = substitutionEntry?.notSucceededBy || []
+		const followedBy = candidateEntry?.followedBy ?? candidateEntry?.succeededBy ?? []
+		const notFollowedBy = candidateEntry?.notFollowedBy ?? candidateEntry?.notSucceededBy ?? []
 
-		const hasNegativePattern = notPrecededBy.includes(precedingWord) || notSucceededBy.includes(succeedingWord)
-		const hasPositivePattern = precededBy.includes(precedingWord) || succeededBy.includes(succeedingWord)
+		const hasNegativePattern = notPrecededBy.includes(precedingWord) || notFollowedBy.includes(followingWord)
+		const hasPositivePattern = precededBy.includes(precedingWord) || followedBy.includes(followingWord)
 
-		if (i == lexiconEntry.length - 1 || (hasPositivePattern && !hasNegativePattern)) {
-			const substitutionPhonemes = substitutionPhonemesText.split(/ +/g)
+		if (i === lexiconEntry.length - 1 || (hasPositivePattern && !hasNegativePattern)) {
+			const substitutionPhonemes = pronunciationPhonemesText.split(/ +/g)
 
 			return substitutionPhonemes
 		}
 	}
 
-	return
+	return undefined
 }
 
 export async function loadLexiconFile(jsonFilePath: string): Promise<Lexicon> {
@@ -115,8 +115,11 @@ export type LexiconEntry = {
 	precededBy?: string[]
 	notPrecededBy?: string[]
 
-	succeededBy?: string[]
-	notSucceededBy?: string[]
+	followedBy?: string[]
+	notFollowedBy?: string[]
+
+	succeededBy?: string[] // Deprecated. Replaced by 'followedBy'
+	notSucceededBy?: string[] // Deprecated. Replaced by 'notFollowedBy'
 
 	example?: string
 }