echogarden 2.4.0 → 2.5.1

package/docs/Options.md

@@ -101,7 +101,8 @@ Applies to CLI operations: `speak`, `speak-file`, `speak-url`, `speak-wikipedia`
 * `openAICloud.apiKey`: API key (required)
 * `openAICloud.organization`: organization identifier. Optional
 * `openAICloud.baseURL`: override the default base URL for the API. Optional
-* `openAICloud.model`: model to use. Can be either `tts-1` or `tts-1-hd`. Defaults to `tts-1`
+* `openAICloud.model`: model to use. Can be either `tts-1`, `tts-1-hd` or `gpt-4o-mini-tts`. Defaults to `tts-1`
+* `openAICloud.instructions`: instructions for model. Only works with `gpt-4o-mini-tts`. Optional
 * `openAICloud.timeout`: request timeout. Optional
 * `openAICloud.maxRetries`: maximum retries on failure. Defaults to 10
 
@@ -213,7 +214,7 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 
 **OpenAI Cloud**:
 * `openAICloud.apiKey`: API key (required)
-* `openAICloud.model`: model to use. When using the default provider (OpenAI), can only be `whisper-1`. For a custom provider, like Groq, see its documentation
+* `openAICloud.model`: model to use. When using the default provider (OpenAI), can be `whisper-1`, `gpt-4o-mini-transcribe` or `gpt-4o-transcribe`. For a custom provider, like Groq, see its documentation
 * `openAICloud.organization`: organization identifier. Optional
 * `openAICloud.baseURL`: override the default endpoint used by the API. For example, set `https://api.groq.com/openai/v1` to use Groq's OpenAI-compatible API instead of the default one. Optional
 * `openAICloud.temperature`: temperature. Choosing `0` uses a dynamic temperature approach. Defaults to `0`

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "2.4.0",
+	"version": "2.5.1",
 	"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -52,8 +52,8 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "^3.772.0",
-		"@aws-sdk/client-transcribe-streaming": "^3.772.0",
+		"@aws-sdk/client-polly": "^3.787.0",
+		"@aws-sdk/client-transcribe-streaming": "^3.787.0",
 		"@echogarden/audio-io": "^0.3.0",
 		"@echogarden/espeak-ng-emscripten": "^0.3.3",
 		"@echogarden/fasttext-wasm": "^0.1.0",
@@ -79,13 +79,13 @@
 		"html-to-text": "^9.0.5",
 		"import-meta-resolve": "^4.1.0",
 		"jieba-wasm": "^2.2.0",
-		"jsdom": "^26.0.0",
+		"jsdom": "^26.1.0",
 		"json5": "^2.2.3",
 		"kuromoji": "^0.1.2",
-		"microsoft-cognitiveservices-speech-sdk": "^1.43.0",
+		"microsoft-cognitiveservices-speech-sdk": "^1.43.1",
 		"msgpack-lite": "^0.1.26",
 		"onnxruntime-node": "^1.21.0",
-		"openai": "^4.89.0",
+		"openai": "^4.95.1",
 		"sam-js": "^0.3.1",
 		"strip-ansi": "^7.1.0",
 		"tar": "^7.4.3",
@@ -112,11 +112,11 @@
 		"@types/graceful-fs": "^4.1.9",
 		"@types/jsdom": "^21.1.7",
 		"@types/msgpack-lite": "^0.1.11",
-		"@types/node": "^22.13.13",
+		"@types/node": "^22.14.1",
 		"@types/tar": "^6.1.13",
-		"@types/ws": "^8.18.0",
-		"ts-json-schema-generator": "^2.3.0",
-		"typescript": "^5.8.2"
+		"@types/ws": "^8.18.1",
+		"ts-json-schema-generator": "^2.4.0",
+		"typescript": "^5.8.3"
 	},
 	"overrides": {
 		"whatwg-url": "^14.0.0"

package/src/cli/CLI.ts

@@ -1698,21 +1698,27 @@ async function writeSourceSeparationOutputIfNeeded(outputFilename: string, isola
 		{
 			const isolatedOutputFilePath = prefixIsolated ? `${pathWithoutExtension}.isolated.${fileExtension}` : outputFilename
 
-			const fileSaver = getFileSaver(isolatedOutputFilePath, allowOverwrite)
-
-			await fileSaver(isolatedRawAudio, [], '')
+			await saveAudioWithBitrate(isolatedRawAudio, isolatedOutputFilePath, fileExtension, 64)
 		}
 
 		{
 			const backgroundOutputFilePath = `${pathWithoutExtension}.background.${fileExtension}`
 
-			const fileSaver = getFileSaver(backgroundOutputFilePath, allowOverwrite)
-
-			await fileSaver(backgroundRawAudio, [], '')
+			await saveAudioWithBitrate(backgroundRawAudio, backgroundOutputFilePath, fileExtension, 128)
 		}
 	}
 }
 
+async function saveAudioWithBitrate(rawAudio: RawAudio, filepath: string, fileExtension: string, bitrate: number) {
+	const ffmpegOptions = getDefaultFFMpegOptionsForSpeech(fileExtension, bitrate)
+	ffmpegOptions.filename = filepath
+
+	const fileDir = parsePath(filepath).dir || './'
+
+	await ensureDir(fileDir)
+	await encodeFromChannels(rawAudio, ffmpegOptions)
+}
+
 async function optionsLookupToTypedObject<K extends keyof APIOptions>(cliOptionsMap: Map<string, string>, optionsRoot: K, additionalOptionsSchema?: Map<string, SchemaTypeDefinition>): Promise<APIOptions[K]> {
 	const optionsSchema = await getOptionsSchema()
 	const resultingObj: any = {}

package/src/nlp/PhoneConversion.ts

@@ -5,7 +5,7 @@ export function ipaPhoneToKirshenbaum(ipaPhone: string) {
 		const convertedChar = ipaToKirshenbaum[char]
 
 		if (convertedChar == undefined) {
-			throw new Error(`Couldn't convert IPA character '${char}' (part of phone '${ipaPhone}' the Kirshenbaum notation`)
+			throw new Error(`Couldn't convert IPA character '${char}' (part of phone '${ipaPhone}') to the Kirshenbaum notation`)
 		}
 
 		result += convertedChar || '_'
@@ -288,8 +288,9 @@ export const ipaToKirshenbaum: Record<string, string> = {
 	// Open-mid:
 	'ɛ': 'E',
 	'œ': 'W',
-	//'ɜ': '3', // eSpeak
-	'ɜ': 'V"',
+	'ɜ': '3', // eSpeak
+	'ɜː': '3:',
+	//'ɜ': 'V"',
 	'ɞ': 'O"',
 	'ʌ': 'V',
 	'ɔ': 'O',

package/src/nlp/TextNormalizer.ts

@@ -1,11 +1,16 @@
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { substituteCharactersUsingLookup } from '../utilities/StringUtilities.js'
 
-export function getNormalizedFragmentsForSpeech(words: string[], language: string) {
+export function getNormalizedFragmentsForSpeech(
+	words: string[],
+	nonWhitespaceWords: string[],
+	nonWhitespaceWordOriginalIndex: number[],
+	language: string) {
+
 	language = getShortLanguageCode(language)
 
 	if (language != 'en') {
-		return { normalizedFragments: [...words], referenceFragments: [...words] }
+		return { normalizedFragments: [...nonWhitespaceWords], referenceFragments: [...nonWhitespaceWords] }
 	}
 
 	const numberPattern = /^[0-9][0-9\,\.]*$/
@@ -41,73 +46,79 @@ export function getNormalizedFragmentsForSpeech(words: string[], language: strin
 	const normalizedFragments: string[] = []
 	const referenceFragments: string[] = []
 
-	for (let wordIndex = 0; wordIndex < words.length; wordIndex++) {
-		const word = words[wordIndex]
+	for (let wordIndex = 0; wordIndex < nonWhitespaceWords.length; wordIndex++) {
+		const word = nonWhitespaceWords[wordIndex]
 		const lowerCaseWord = word.toLowerCase()
 
-		const nextWords = words.slice(wordIndex + 1)
-		const nextWord = nextWords[0]
+		const nextNonWhitespaceWords = nonWhitespaceWords.slice(wordIndex + 1)
+		const nextNonWhitespaceWord = nextNonWhitespaceWords[0]
+
+		const originalWordIndex = nonWhitespaceWordOriginalIndex[wordIndex]
+		const isFollowedByWhitespace = words[originalWordIndex + 1]?.trim().length === 0
 
 		if ( // Normalize a four digit year pattern, e.g. 'in 1995'.
 			wordsPrecedingAYear.includes(lowerCaseWord) &&
-			fourDigitYearPattern.test(nextWord)) {
+			isFollowedByWhitespace &&
+			fourDigitYearPattern.test(nextNonWhitespaceWord)) {
 
-			const normalizedString = normalizeFourDigitYearString(nextWord)
+			const normalizedString = normalizeFourDigitYearString(nextNonWhitespaceWord)
 
 			normalizedFragments.push(word)
 			referenceFragments.push(word)
 
 			normalizedFragments.push(normalizedString)
-			referenceFragments.push(nextWord)
+			referenceFragments.push(nextNonWhitespaceWord)
 
 			wordIndex += 1
 		} else if ( // Normalize a four digit decade pattern, e.g. 'the 1980s'.
 			wordsPrecedingADecade.includes(lowerCaseWord) &&
-			fourDigitDecadePattern.test(nextWord)) {
+			isFollowedByWhitespace &&
+			fourDigitDecadePattern.test(nextNonWhitespaceWord)) {
 
-			const normalizedString = normalizeFourDigitDecadeString(nextWord)
+			const normalizedString = normalizeFourDigitDecadeString(nextNonWhitespaceWord)
 
 			normalizedFragments.push(word)
 			referenceFragments.push(word)
 
 			normalizedFragments.push(normalizedString)
-			referenceFragments.push(nextWord)
+			referenceFragments.push(nextNonWhitespaceWord)
 
 			wordIndex += 1
-		} else if ( // Normalize a year range pattern, e.g. '1835-1896'
-			fourDigitYearRangePattern.test(words.slice(wordIndex, wordIndex + 3).join(''))) {
+		} else if ( // Normalize a year range pattern, e.g. '1835-1896', ensure there are no spaces between words
+			fourDigitYearRangePattern.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
 
-			normalizedFragments.push(normalizeFourDigitYearString(words[wordIndex]))
-			referenceFragments.push(words[wordIndex])
+			normalizedFragments.push(normalizeFourDigitYearString(nonWhitespaceWords[wordIndex]))
+			referenceFragments.push(nonWhitespaceWords[wordIndex])
 
 			normalizedFragments.push('to')
-			referenceFragments.push(words[wordIndex + 1])
+			referenceFragments.push(nonWhitespaceWords[wordIndex + 1])
 
-			normalizedFragments.push(normalizeFourDigitYearString(words[wordIndex + 2]))
-			referenceFragments.push(words[wordIndex + 2])
+			normalizedFragments.push(normalizeFourDigitYearString(nonWhitespaceWords[wordIndex + 2]))
+			referenceFragments.push(nonWhitespaceWords[wordIndex + 2])
 
 			wordIndex += 2
 		} else if ( // Normalize a currency pattern, e.g. '$53.1 million', '€3.53'
 			symbolsPrecedingACurrency.includes(lowerCaseWord) &&
-			numberPattern.test(nextWord)) {
+			!isFollowedByWhitespace &&
+			numberPattern.test(nextNonWhitespaceWord)) {
 
 			let currencyWord = symbolsPrecedingACurrencyAsWords[symbolsPrecedingACurrency.indexOf(lowerCaseWord)]
 
-			if (wordsSucceedingACurrency.includes(nextWords[1].toLowerCase())) {
-				const normalizedString = `${nextWord} ${nextWords[1]} ${currencyWord}`
+			if (wordsSucceedingACurrency.includes(nextNonWhitespaceWords[1]?.toLowerCase())) {
+				const normalizedString = `${nextNonWhitespaceWord} ${nextNonWhitespaceWords[1]} ${currencyWord}`
 
 				normalizedFragments.push(normalizedString)
 
-				const referenceString = `${word}${nextWord} ${nextWords[1]}`
+				const referenceString = `${word}${nextNonWhitespaceWord} ${nextNonWhitespaceWords[1]}`
 				referenceFragments.push(referenceString)
 
 				wordIndex += 2
 			} else {
-				const normalizedString = `${nextWord} ${currencyWord}`
+				const normalizedString = `${nextNonWhitespaceWord} ${currencyWord}`
 
 				normalizedFragments.push(normalizedString)
 
-				const referenceString = `${word}${nextWord}`
+				const referenceString = `${word}${nextNonWhitespaceWord}`
 				referenceFragments.push(referenceString)
 
 				wordIndex += 1

package/src/recognition/OpenAICloudSTT.ts

@@ -35,6 +35,14 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 
 	logger.start(options.baseURL ? `Send request to ${options.baseURL}` : 'Send request to OpenAI Cloud API')
 
+	let responseFormat: 'verbose_json' | 'json'
+
+	if (options.model === 'gpt-4o-mini-transcribe' || options.model === 'gpt-4o-transcribe') {
+		responseFormat = 'json'
+	} else {
+		responseFormat = 'verbose_json'
+	}
+
 	let response: VerboseResponse
 
 	if (task == 'transcribe') {
@@ -46,7 +54,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			model: options.model,
 			language: languageCode,
 			prompt: options.prompt,
-			response_format: 'verbose_json',
+			response_format: responseFormat,
 			temperature: options.temperature,
 			timestamp_granularities,
 		}) as any as VerboseResponse
@@ -55,7 +63,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			file: virtualFileStream,
 			model: options.model,
 			prompt: options.prompt,
-			response_format: 'verbose_json',
+			response_format: responseFormat,
 			temperature: options.temperature,
 		}) as any as VerboseResponse
 	} else {
@@ -64,7 +72,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 
 	const transcript = response.text.trim()
 
-	let timeline: Timeline
+	let timeline: Timeline | undefined
 
 	if (response.words) {
 		timeline = response.words.map<TimelineEntry>(entry => ({
@@ -73,7 +81,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			startTime: entry.start,
 			endTime: entry.end
 		}))
-	} else {
+	} else if (response.segments) {
 		const segmentTimeline = response.segments.map<TimelineEntry>(entry => ({
 			type: 'segment',
 			text: entry.text,
@@ -129,7 +137,7 @@ interface VerboseResponse {
 type Task = 'transcribe' | 'translate'
 
 export interface OpenAICloudSTTOptions {
-	model?: 'whisper-1' | string
+	model?: 'whisper-1' | 'gpt-4o-mini-transcribe' | 'gpt-4o-transcribe' | string
 
 	apiKey?: string
 	organization?: string

package/src/synthesis/EspeakTTS.ts

@@ -40,30 +40,45 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 	let words = await splitToWords(text, language)
 
 	// Merge repeating non-words to a single word to work around eSpeak bug
-	const wordsWithMerges: string[] = []
+	{
+		const wordsWithMerges: string[] = []
 
-	for (let i = 0; i < words.length; i++) {
-		const currentWord = words[i]
-		const previousWord = words[i - 1]
+		for (let i = 0; i < words.length; i++) {
+			const currentWord = words[i]
+			const previousWord = words[i - 1]
 
-		if (
-			i > 0 &&
-			currentWord === previousWord &&
-			!['[', ']'].includes(currentWord) && // Work around eSpeak-NG marker bug with repeating squared brackets
-			!wordCharacterPattern.test(currentWord)) {
+			if (
+				i > 0 &&
+				currentWord === previousWord &&
+				!['[', ']'].includes(currentWord) && // Work around eSpeak-NG marker bug with repeating squared brackets
+				!wordCharacterPattern.test(currentWord)) {
 
-			wordsWithMerges[wordsWithMerges.length - 1] += currentWord
-		} else {
-			wordsWithMerges.push(currentWord)
+				wordsWithMerges[wordsWithMerges.length - 1] += currentWord
+			} else {
+				wordsWithMerges.push(currentWord)
+			}
 		}
+
+		words = wordsWithMerges
 	}
 
-	words = wordsWithMerges
+	// Remove words containing only whitespace
+	const nonWhitespaceWords: string[] = []
+	const nonWhitespaceWordsOriginalIndex: number[] = []
 
-	// Trim words and remove words containing only whitespace
-	words = words.map(word => word.trim()).filter(word => word !== '')
+	{
+		for (let i = 0; i < words.length; i++) {
+			const word = words[i]
+			const wordIsWhitespace = word.trim().length === 0
+
+			if (!wordIsWhitespace) {
+				nonWhitespaceWords.push(word)
+				nonWhitespaceWordsOriginalIndex.push(i)
+			}
+		}
+	}
 
-	const { normalizedFragments, referenceFragments } = getNormalizedFragmentsForSpeech(words, language)
+	const { normalizedFragments, referenceFragments } = getNormalizedFragmentsForSpeech(words, nonWhitespaceWords, nonWhitespaceWordsOriginalIndex, language)
 
 	const simplifiedFragments = normalizedFragments.map(word => simplifyPunctuationCharacters(word).toLocaleLowerCase())
 

package/src/synthesis/OpenAICloudTTS.ts

@@ -22,6 +22,7 @@ export async function synthesize(text: string, voice: string, speed: number, opt
 		input: text,
 		model: options.model!,
 		voice: voice as any,
+		instructions: options.instructions,
 		response_format: 'opus',
 		speed,
 	}, {
@@ -43,7 +44,9 @@ export interface OpenAICloudTTSOptions {
 	organization?: string
 	baseURL?: string
 
-	model?: 'tts-1' | 'tts-1-hd'
+	model?: 'tts-1' | 'tts-1-hd' | 'gpt-4o-mini-tts'
+
+	instructions?: string
 
 	timeout?: number
 	maxRetries?: number
@@ -56,110 +59,72 @@ export const defaultOpenAICloudTTSOptions: OpenAICloudTTSOptions = {
 
 	model: 'tts-1',
 
+	instructions: undefined,
+
 	timeout: undefined,
 	maxRetries: 10,
 }
 
+// Reference: https://platform.openai.com/docs/guides/text-to-speech#supported-languages
 export const supportedLanguages = [
 	'en',
-	'zh',
-	'de',
-	'es',
-	'ru',
-	'ko',
-	'fr',
-	'ja',
-	'pt',
-	'tr',
-	'pl',
+	'af',
+	'ar',
+	'hy',
+	'az',
+	'be',
+	'bs',
+	'bg',
 	'ca',
+	'zh',
+	'hr',
+	'cs',
+	'da',
 	'nl',
-	'ar',
-	'sv',
-	'it',
-	'id',
-	'hi',
+	'en',
+	'et',
 	'fi',
-	'vi',
-	'iw',
-	'uk',
+	'fr',
+	'gl',
+	'de',
 	'el',
-	'ms',
-	'cs',
-	'ro',
-	'da',
+	'he',
+	'hi',
 	'hu',
-	'ta',
-	'no',
-	'th',
-	'ur',
-	'hr',
-	'bg',
+	'is',
+	'id',
+	'it',
+	'ja',
+	'kn',
+	'kk',
+	'ko',
+	'lv',
 	'lt',
-	'la',
+	'mk',
+	'ms',
+	'mr',
 	'mi',
-	'ml',
-	'cy',
-	'sk',
-	'te',
+	'ne',
+	'no',
 	'fa',
-	'lv',
-	'bn',
+	'pl',
+	'pt',
+	'ro',
+	'ru',
 	'sr',
-	'az',
+	'sk',
 	'sl',
-	'kn',
-	'et',
-	'mk',
-	'br',
-	'eu',
-	'is',
-	'hy',
-	'ne',
-	'mn',
-	'bs',
-	'kk',
-	'sq',
+	'es',
 	'sw',
-	'gl',
-	'mr',
-	'pa',
-	'si',
-	'km',
-	'sn',
-	'yo',
-	'so',
-	'af',
-	'oc',
-	'ka',
-	'be',
-	'tg',
-	'sd',
-	'gu',
-	'am',
-	'yi',
-	'lo',
-	'uz',
-	'fo',
-	'ht',
-	'ps',
-	'tk',
-	'nn',
-	'mt',
-	'sa',
-	'lb',
-	'my',
-	'bo',
+	'sv',
 	'tl',
-	'mg',
-	'as',
-	'tt',
-	'haw',
-	'ln',
-	'ha',
-	'ba',
-	'jw',
-	'su',
+	'ta',
+	'th',
+	'tr',
+	'uk',
+	'ur',
+	'vi',
+	'cy',
 ]
 
 export const voiceList: SynthesisVoice[] = [
@@ -173,6 +138,11 @@ export const voiceList: SynthesisVoice[] = [
 		languages: ['en-US', ...supportedLanguages],
 		gender: 'male',
 	},
+	{
+		name: 'ballad',
+		languages: ['en-GB', ...supportedLanguages],
+		gender: 'male',
+	},
 	{
 		name: 'coral',
 		languages: ['en-US', ...supportedLanguages],
@@ -208,4 +178,9 @@ export const voiceList: SynthesisVoice[] = [
 		languages: ['en-US', ...supportedLanguages],
 		gender: 'female',
 	},
+	{
+		name: 'verse',
+		languages: ['en-US', ...supportedLanguages],
+		gender: 'male',
+	},
 ]