echogarden 2.2.0 → 2.3.0

package/src/api/Synthesis.ts

@@ -22,6 +22,7 @@ import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 import { type EspeakOptions } from '../synthesis/EspeakTTS.js'
 import { type OpenAICloudTTSOptions } from '../synthesis/OpenAICloudTTS.js'
 import { type ElevenLabsTTSOptions } from '../synthesis/ElevenLabsTTS.js'
+import { type DeepgramTTSOptions } from '../synthesis/DeepgramTTS.js'
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 import { simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
 import { convertHtmlToText } from '../utilities/StringUtilities.js'
@@ -808,23 +809,50 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 		case 'elevenlabs': {
 			if (inputIsSSML) {
-				throw new Error(`The Elevenlabs engine doesn't support SSML inputs`)
+				throw new Error(`The ElevenLabs engine doesn't support SSML inputs`)
 			}
 
 			const ElevenLabsTTS = await import('../synthesis/ElevenLabsTTS.js')
 
-			const engineOptions = options.elevenlabs!
+			const engineOptions = options.elevenLabs!
 
 			if (!engineOptions.apiKey) {
 				throw new Error(`No ElevenLabs API key provided`)
 			}
 
 			const voiceId = (selectedVoice as any)['elevenLabsVoiceId']
-			const modelId = (selectedVoice as any)['elevenLabsModelId']
 
 			logger.end()
 
-			const { rawAudio } = await ElevenLabsTTS.synthesize(text, voiceId, modelId, engineOptions)
+			const { rawAudio, timeline: outTimeline } = await ElevenLabsTTS.synthesize(text, voiceId, language, engineOptions)
+
+			synthesizedAudio = rawAudio
+			timeline = outTimeline
+
+			shouldPostprocessSpeed = true
+			shouldPostprocessPitch = true
+
+			break
+		}
+
+		case 'deepgram': {
+			if (inputIsSSML) {
+				throw new Error(`The Deepgram engine doesn't support SSML inputs`)
+			}
+
+			const DeepgramTTS = await import('../synthesis/DeepgramTTS.js')
+
+			const engineOptions = options.deepgram!
+
+			if (!engineOptions.apiKey) {
+				throw new Error(`No Deepgram API key provided`)
+			}
+
+			const modelId = selectedVoice.deepgramModelId
+
+			logger.end()
+
+			const { rawAudio } = await DeepgramTTS.synthesize(text, modelId, engineOptions)
 
 			synthesizedAudio = rawAudio
 
@@ -1071,8 +1099,8 @@ export type SynthesisEngine =
 	'vits' | 'kokoro' | 'pico' | 'flite' | 'gnuspeech' |
 	'espeak' | 'sam' | 'sapi' | 'msspeech' | 'coqui-server' |
 	'google-cloud' | 'microsoft-azure' | 'amazon-polly' |
-	'openai-cloud' | 'elevenlabs' | 'google-translate' |
-	'microsoft-edge' | 'streamlabs-polly'
+	'openai-cloud' | 'elevenlabs' | 'deepgram' |
+	'google-translate' | 'microsoft-edge' | 'streamlabs-polly'
 
 export type TimePitchShiftingMethod = 'sonic' | 'rubberband'
 
@@ -1198,7 +1226,9 @@ export interface SynthesisOptions {
 
 	openAICloud?: OpenAICloudTTSOptions
 
-	elevenlabs?: ElevenLabsTTSOptions,
+	elevenLabs?: ElevenLabsTTSOptions,
+
+	deepgram?: DeepgramTTSOptions
 
 	googleTranslate?: {
 		tld?: string
@@ -1340,7 +1370,10 @@ export const defaultSynthesisOptions: SynthesisOptions = {
 	openAICloud: {
 	},
 
-	elevenlabs: {
+	elevenLabs: {
+	},
+
+	deepgram: {
 	},
 
 	googleTranslate: {
@@ -1595,7 +1628,7 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 			case 'elevenlabs': {
 				const ElevenLabsTTS = await import('../synthesis/ElevenLabsTTS.js')
 
-				const engineOptions = options.elevenlabs!
+				const engineOptions = options.elevenLabs!
 
 				const apiKey = engineOptions.apiKey
 
@@ -1608,6 +1641,14 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 				break
 			}
 
+			case 'deepgram': {
+				const DeepgramTTS = await import('../synthesis/DeepgramTTS.js')
+
+				voiceList = DeepgramTTS.voiceList
+
+				break
+			}
+
 			case 'google-translate': {
 				const GoogleTranslateTTS = await import('../synthesis/GoogleTranslateTTS.js')
 
@@ -1800,6 +1841,7 @@ export interface SynthesisVoice {
 	gender: VoiceGender
 	speakerCount?: number
 	packageName?: string
+	[key: string]: any
 }
 
 export type VoiceGender = 'male' | 'female' | 'unknown'
@@ -1891,7 +1933,13 @@ export const synthesisEngines: EngineMetadata[] = [
 	},
 	{
 		id: 'elevenlabs',
-		name: 'Elevenlabs',
+		name: 'ElevenLabs',
+		description: 'A generative AI text-to-speech cloud service.',
+		type: 'cloud'
+	},
+	{
+		id: 'deepgram',
+		name: 'Deepgram',
 		description: 'A generative AI text-to-speech cloud service.',
 		type: 'cloud'
 	},

package/src/recognition/DeepgramSTT.ts

@@ -1,4 +1,4 @@
-import { request } from 'gaxios'
+import { GaxiosResponse, request } from 'gaxios'
 import { RawAudio } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
@@ -18,9 +18,9 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 
 	// Prepare API request
 	const params: Record<string, string> = {
-		model: options.model || 'whisper-large',
-		encoding: 'flac',
-		//punctuate: 'true', // Problem when enabled: words in timeline are not capitalized, causing errors
+		model: options.model!,
+		encoding: 'opus',
+		punctuate: options.punctuate ? 'true' : 'false',
 	}
 
 	// Set language or enable auto-detection
@@ -31,32 +31,49 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 	}
 
 	// Set audio encoding parameters
-	logger.start('Convert audio to FLAC format')
+	logger.start('Convert audio to Opus format')
 
 	const audioData = await FFMpegTranscoder.encodeFromChannels(
 		rawAudio,
-		FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('flac')
+		FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('opus')
 	)
 
 	logger.start('Send request to Deepgram API')
 
-	const response = await request<any>({
-		method: 'POST',
+	let response: GaxiosResponse<any>
 
-		url: 'https://api.deepgram.com/v1/listen',
+	try {
+		response = await request<any>({
+			method: 'POST',
 
-		params,
+			url: 'https://api.deepgram.com/v1/listen',
 
-		headers: {
-			'Authorization': `Token ${options.apiKey}`,
-			'Content-Type': 'audio/flac',
-			'Accept': 'application/json'
-		},
+			params,
 
-		body: audioData,
+			headers: {
+				'Authorization': `Token ${options.apiKey}`,
+				'Content-Type': 'audio/ogg',
+				'Accept': 'application/json'
+			},
 
-		responseType: 'json',
-	})
+			body: audioData,
+
+			responseType: 'json',
+		})
+	} catch (e: any) {
+		const response = e.response
+
+		if (response) {
+			logger.log(`Request failed with status code ${response.status}`)
+
+			if (response.data) {
+				logger.log(`Server responded with:`)
+				logger.log(response.data)
+			}
+		}
+
+		throw e
+	}
 
 	const deepgramResponse: DeepgramResponse = response.data
 
@@ -65,19 +82,37 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 	// Extract transcript and create timeline
 	const transcript = firstAlternative?.transcript || ''
 
-	let timeline: Timeline = []
-
 	// Extract word-level timing information if available
 	const words = firstAlternative?.words || []
 
-	if (words.length > 0) {
-		timeline = words.map((word: DeepgramWordEntry): TimelineEntry => ({
-			type: 'word',
-			text: word.word,
-			startTime: word.start,
-			endTime: word.end,
-			confidence: word.confidence,
-		}))
+	const timeline = words.map((wordEntry: DeepgramWordEntry) => ({
+		type: 'word',
+		text: wordEntry.word,
+		startTime: wordEntry.start,
+		endTime: wordEntry.end,
+		confidence: wordEntry.confidence,
+	} as TimelineEntry))
+
+	// If `punctuate` is set to `true`, modify the text of all words to match their exact case in the transcript.
+	// This is required, otherwise it would later fail deriving word offsets.
+	if (options.punctuate) {
+		const lowerCaseTranscript = transcript.toLocaleLowerCase()
+
+		let readOffset = 0
+
+		for (const wordEntry of timeline) {
+			const wordEntryTextLowercase = wordEntry.text.toLocaleLowerCase()
+
+			const matchPosition = lowerCaseTranscript.indexOf(wordEntryTextLowercase, readOffset)
+
+			if (matchPosition === -1) {
+				throw new Error(`Couldn't match the word '${wordEntry.text}' in the lowercase transcript`)
+			}
+
+			wordEntry.text = transcript.substring(matchPosition, matchPosition + wordEntryTextLowercase.length)
+
+			readOffset = matchPosition + wordEntry.text.length
+		}
 	}
 
 	logger.end()
@@ -88,11 +123,13 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 export interface DeepgramSTTOptions {
 	apiKey?: string
 	model?: string
+	punctuate?: boolean
 }
 
 export const defaultDeepgramSTTOptions: DeepgramSTTOptions = {
 	apiKey: undefined,
-	model: 'nova-2'
+	model: 'nova-2',
+	punctuate: true,
 }
 
 interface DeepgramWordEntry {

package/src/synthesis/DeepgramTTS.ts

@@ -0,0 +1,149 @@
+import { GaxiosResponse, request } from 'gaxios'
+import { SynthesisVoice } from '../api/API.js'
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Logger } from '../utilities/Logger.js'
+import { logToStderr } from '../utilities/Utilities.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+const log = logToStderr
+
+export async function synthesize(text: string, modelId: string, options: DeepgramTTSOptions) {
+	const logger = new Logger()
+	logger.start('Request synthesis from Deepgram')
+
+	options = extendDeep(defaultDeepgramTTSOptions, options)
+
+	let response: GaxiosResponse<any>
+
+	try {
+		response = await request<any>({
+			url: `https://api.deepgram.com/v1/speak`,
+
+			params: {
+				model: modelId,
+				encoding: 'mp3',
+				bit_rate: 48000,
+			},
+
+			method: 'POST',
+
+			headers: {
+				'Content-Type': 'application/json',
+				'Authorization': `Token ${options.apiKey}`,
+			},
+
+			data: {
+				text,
+			},
+
+			responseType: 'arraybuffer'
+		})
+	} catch (e: any) {
+		const response = e.response
+
+		if (response) {
+			logger.log(`Request failed with status code ${response.status}`)
+
+			if (response.data) {
+				logger.log(`Server responded with:`)
+				logger.log(response.data)
+			}
+		}
+
+		throw e
+	}
+
+	logger.start('Decode synthesized audio')
+	const rawAudio = await FFMpegTranscoder.decodeToChannels(new Uint8Array(response.data))
+
+	logger.end()
+
+	return { rawAudio }
+}
+
+export async function getVoiceList() {
+	return voiceList
+}
+
+export interface DeepgramTTSOptions {
+	apiKey?: string
+}
+
+export const defaultDeepgramTTSOptions = {
+	apiKey: undefined,
+}
+
+export const voiceList: SynthesisVoice[] = [
+	{
+		name: 'Asteria',
+		deepgramModelId: 'aura-asteria-en',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Luna',
+		deepgramModelId: 'aura-luna-en',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Stella',
+		deepgramModelId: 'aura-stella-en',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Athena',
+		deepgramModelId: 'aura-athena-en',
+		languages: ['en-GB', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Hera',
+		deepgramModelId: 'aura-hera-en',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Orion',
+		deepgramModelId: 'aura-orion-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Arcas',
+		deepgramModelId: 'aura-arcas-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Perseus',
+		deepgramModelId: 'aura-perseus-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Angus',
+		deepgramModelId: 'aura-angus-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Orpheus',
+		deepgramModelId: 'aura-orpheus-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Helios',
+		deepgramModelId: 'aura-helios-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Zeus',
+		deepgramModelId: 'aura-zeus-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+]

package/src/synthesis/ElevenLabsTTS.ts

@@ -4,10 +4,13 @@ import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { decodeBase64 } from '../encodings/Base64.js'
+import { isWordOrSymbolWord, splitToWords } from '../nlp/Segmentation.js'
+import { Timeline } from '../utilities/Timeline.js'
 
 const log = logToStderr
 
-export async function synthesize(text: string, voiceId: string, modelId: string, options: ElevenLabsTTSOptions) {
+export async function synthesize(text: string, voiceId: string, language: string, options: ElevenLabsTTSOptions) {
 	const logger = new Logger()
 	logger.start('Request synthesis from ElevenLabs')
 
@@ -17,7 +20,7 @@ export async function synthesize(text: string, voiceId: string, modelId: string,
 
 	try {
 		response = await request<any>({
-			url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`,
+			url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/with-timestamps`,
 
 			method: 'POST',
 
@@ -26,20 +29,26 @@ export async function synthesize(text: string, voiceId: string, modelId: string,
 				'xi-api-key': options.apiKey,
 			},
 
+			params: {
+				output_format: 'mp3_44100_64',
+			},
+
 			data: {
 				text,
 
-				model_id: modelId,
+				model_id: options.modelId,
 
 				voice_setting: {
 					stability: options.stability,
 					similarity_boost: options.similarityBoost,
 					style: options.style,
-					use_speaker_boost: options.useSpeakerBoost
-				}
+					use_speaker_boost: options.useSpeakerBoost,
+				},
+
+				seed: options.seed
 			},
 
-			responseType: 'arraybuffer'
+			responseType: 'json'
 		})
 	} catch (e: any) {
 		const response = e.response
@@ -57,11 +66,43 @@ export async function synthesize(text: string, voiceId: string, modelId: string,
 	}
 
 	logger.start('Decode synthesized audio')
-	const rawAudio = await FFMpegTranscoder.decodeToChannels(new Uint8Array(response.data))
+	const audioData = decodeBase64(response.data.audio_base64)
+	const rawAudio = await FFMpegTranscoder.decodeToChannels(audioData)
+
+	let timeline: Timeline | undefined
+
+	const characters: string[] = response.data.alignment?.characters
+	const characterStartTimes: number[] = response.data.alignment?.character_start_times_seconds
+	const characterEndTimes: number[] = response.data.alignment?.character_end_times_seconds
+
+	if (characters && characterStartTimes && characterEndTimes) {
+		logger.start('Create timeline from returned character timings')
+
+		const referenceText = characters.join('')
+		const words = (await splitToWords(referenceText, language)).filter(w => isWordOrSymbolWord(w))
+
+		timeline = []
+
+		let offset = 0
+
+		for (const word of words) {
+			const wordStartIndex = referenceText.indexOf(word, offset)
+			const wordEndIndex = wordStartIndex + word.length
+
+			timeline.push({
+				type: 'word',
+				text: word,
+				startTime: characterStartTimes[wordStartIndex],
+				endTime: characterEndTimes[wordEndIndex] ?? characterEndTimes[wordEndIndex - 1]
+			})
+
+			offset = wordEndIndex
+		}
+	}
 
 	logger.end()
 
-	return { rawAudio }
+	return { rawAudio, timeline }
 }
 
 export async function getVoiceList(apiKey: string) {
@@ -78,40 +119,36 @@ export async function getVoiceList(apiKey: string) {
 		responseType: 'json'
 	})
 
-	const elevenlabsVoices: any[] = response.data.voices
+	const elevenLabsVoices: any[] = response.data.voices
 
-	const voices: SynthesisVoice[] = elevenlabsVoices.map(elevenlabsVoice => {
-		const modelId: string = elevenlabsVoice?.high_quality_base_model_ids?.[0] ?? 'eleven_monolingual_v1'
-		const accent: string | undefined = elevenlabsVoice?.labels?.accent
-		const gender: VoiceGender = elevenlabsVoice?.labels?.gender ?? 'unknown'
+	const voices: SynthesisVoice[] = elevenLabsVoices.map(elevenLabsVoice => {
+		const gender: VoiceGender = elevenLabsVoice?.labels?.gender ?? 'unknown'
 
 		const supportedLanguages: string[] = []
 
-		if (accent) {
-			if (accent.startsWith('american')) {
-				supportedLanguages.push('en-US')
-			} else if (accent.startsWith('british')) {
-				supportedLanguages.push('en-GB')
-			} else if (accent === 'irish') {
-				supportedLanguages.push('en-IE')
-			} else if (accent == 'australian') {
-				supportedLanguages.push('en-AU')
-			}
-		}
-
-		if (modelId.includes('multilingual')) {
-			supportedLanguages.push('en', ...supporteMultilingualLanguages)
+		let accent: string | undefined = elevenLabsVoice?.labels?.accent
+		accent = accent?.toLowerCase() ?? ''
+
+		if (accent.startsWith('american')) {
+			supportedLanguages.push('en-US')
+		} else if (accent.startsWith('british')) {
+			supportedLanguages.push('en-GB')
+		} else if (accent === 'irish') {
+			supportedLanguages.push('en-IE')
+		} else if (accent == 'australian') {
+			supportedLanguages.push('en-AU')
 		} else {
 			supportedLanguages.push('en')
 		}
 
+		supportedLanguages.push(...supportedLanguagesInMultilingualModels)
+
 		return {
-			name: elevenlabsVoice.name,
+			name: elevenLabsVoice.name,
 			languages: supportedLanguages,
 			gender,
 
-			elevenLabsVoiceId: elevenlabsVoice.voice_id,
-			elevenLabsModelId: modelId
+			elevenLabsVoiceId: elevenLabsVoice.voice_id,
 		}
 	})
 
@@ -120,18 +157,58 @@ export async function getVoiceList(apiKey: string) {
 
 export interface ElevenLabsTTSOptions {
 	apiKey?: string
+	modelId?: string
+
 	stability?: number
 	similarityBoost?: number
 	style?: number
 	useSpeakerBoost?: boolean
+
+	seed?: number
 }
 
 export const defaultElevenLabsTTSOptions = {
 	apiKey: undefined,
+	modelId: 'eleven_multilingual_v2',
+
 	stability: 0.5,
 	similarityBoost: 0.5,
 	style: 0,
-	useSpeakerBoost: true
+	useSpeakerBoost: true,
+
+	seed: undefined,
 }
 
-export const supporteMultilingualLanguages = ['zh', 'ko', 'nl', 'tr', 'sv', 'id', 'tl', 'ja', 'uk', 'el', 'cs', 'fi', 'ro', 'ru', 'da', 'bg', 'ms', 'sk', 'hr', 'ar', 'ta', 'pl', 'de', 'es', 'fr', 'it', 'hi', 'pt']
+export const supportedLanguagesInMultilingualModels = [
+	'ja',
+	'zh',
+	'de',
+	'hi',
+	'fr',
+	'ko',
+	'pt',
+	'it',
+	'es',
+	'id',
+	'nl',
+	'tr',
+	'fil',
+	'pl',
+	'sv',
+	'bg',
+	'ro',
+	'ar',
+	'cs',
+	'el',
+	'fi',
+	'hr',
+	'ms',
+	'sk',
+	'da',
+	'ta',
+	'uk',
+	'ru',
+	'hu',
+	'no',
+	'vi',
+]

package/src/synthesis/EspeakTTS.ts

@@ -170,7 +170,11 @@ export async function synthesizeFragments(fragments: string[], espeakOptions: Es
 		fragment = simplifyPunctuationCharacters(fragment)
 
 		//fragment = encodeHTMLAngleBrackets(fragment)
-		fragment = fragment.replaceAll('<', '_').replaceAll('>', '_').replaceAll(':', ',')
+		fragment = fragment.replaceAll('<', '_').replaceAll('>', '_')
+
+		if (fragment.split('').every(c => c === ':')) {
+			fragment = ','
+		}
 
 		if (espeakOptions.insertSeparators && canInsertSeparators) {
 			const separator = ` | `