echogarden 2.2.1 → 2.3.0

package/src/recognition/DeepgramSTT.ts

@@ -1,4 +1,4 @@
-import { request } from 'gaxios'
+import { GaxiosResponse, request } from 'gaxios'
 import { RawAudio } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
@@ -18,9 +18,9 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 
 	// Prepare API request
 	const params: Record<string, string> = {
-		model: options.model || 'whisper-large',
-		encoding: 'flac',
-		//punctuate: 'true', // Problem when enabled: words in timeline are not capitalized, causing errors
+		model: options.model!,
+		encoding: 'opus',
+		punctuate: options.punctuate ? 'true' : 'false',
 	}
 
 	// Set language or enable auto-detection
@@ -31,32 +31,49 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 	}
 
 	// Set audio encoding parameters
-	logger.start('Convert audio to FLAC format')
+	logger.start('Convert audio to Opus format')
 
 	const audioData = await FFMpegTranscoder.encodeFromChannels(
 		rawAudio,
-		FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('flac')
+		FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('opus')
 	)
 
 	logger.start('Send request to Deepgram API')
 
-	const response = await request<any>({
-		method: 'POST',
+	let response: GaxiosResponse<any>
 
-		url: 'https://api.deepgram.com/v1/listen',
+	try {
+		response = await request<any>({
+			method: 'POST',
 
-		params,
+			url: 'https://api.deepgram.com/v1/listen',
 
-		headers: {
-			'Authorization': `Token ${options.apiKey}`,
-			'Content-Type': 'audio/flac',
-			'Accept': 'application/json'
-		},
+			params,
 
-		body: audioData,
+			headers: {
+				'Authorization': `Token ${options.apiKey}`,
+				'Content-Type': 'audio/ogg',
+				'Accept': 'application/json'
+			},
 
-		responseType: 'json',
-	})
+			body: audioData,
+
+			responseType: 'json',
+		})
+	} catch (e: any) {
+		const response = e.response
+
+		if (response) {
+			logger.log(`Request failed with status code ${response.status}`)
+
+			if (response.data) {
+				logger.log(`Server responded with:`)
+				logger.log(response.data)
+			}
+		}
+
+		throw e
+	}
 
 	const deepgramResponse: DeepgramResponse = response.data
 
@@ -65,19 +82,37 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 	// Extract transcript and create timeline
 	const transcript = firstAlternative?.transcript || ''
 
-	let timeline: Timeline = []
-
 	// Extract word-level timing information if available
 	const words = firstAlternative?.words || []
 
-	if (words.length > 0) {
-		timeline = words.map((word: DeepgramWordEntry): TimelineEntry => ({
-			type: 'word',
-			text: word.word,
-			startTime: word.start,
-			endTime: word.end,
-			confidence: word.confidence,
-		}))
+	const timeline = words.map((wordEntry: DeepgramWordEntry) => ({
+		type: 'word',
+		text: wordEntry.word,
+		startTime: wordEntry.start,
+		endTime: wordEntry.end,
+		confidence: wordEntry.confidence,
+	} as TimelineEntry))
+
+	// If `punctuate` is set to `true`, modify the text of all words to match their exact case in the transcript.
+	// This is required, otherwise it would later fail deriving word offsets.
+	if (options.punctuate) {
+		const lowerCaseTranscript = transcript.toLocaleLowerCase()
+
+		let readOffset = 0
+
+		for (const wordEntry of timeline) {
+			const wordEntryTextLowercase = wordEntry.text.toLocaleLowerCase()
+
+			const matchPosition = lowerCaseTranscript.indexOf(wordEntryTextLowercase, readOffset)
+
+			if (matchPosition === -1) {
+				throw new Error(`Couldn't match the word '${wordEntry.text}' in the lowercase transcript`)
+			}
+
+			wordEntry.text = transcript.substring(matchPosition, matchPosition + wordEntryTextLowercase.length)
+
+			readOffset = matchPosition + wordEntry.text.length
+		}
 	}
 
 	logger.end()
@@ -88,11 +123,13 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 export interface DeepgramSTTOptions {
 	apiKey?: string
 	model?: string
+	punctuate?: boolean
 }
 
 export const defaultDeepgramSTTOptions: DeepgramSTTOptions = {
 	apiKey: undefined,
-	model: 'nova-2'
+	model: 'nova-2',
+	punctuate: true,
 }
 
 interface DeepgramWordEntry {

package/src/synthesis/DeepgramTTS.ts

@@ -0,0 +1,149 @@
+import { GaxiosResponse, request } from 'gaxios'
+import { SynthesisVoice } from '../api/API.js'
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Logger } from '../utilities/Logger.js'
+import { logToStderr } from '../utilities/Utilities.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+const log = logToStderr
+
+export async function synthesize(text: string, modelId: string, options: DeepgramTTSOptions) {
+	const logger = new Logger()
+	logger.start('Request synthesis from Deepgram')
+
+	options = extendDeep(defaultDeepgramTTSOptions, options)
+
+	let response: GaxiosResponse<any>
+
+	try {
+		response = await request<any>({
+			url: `https://api.deepgram.com/v1/speak`,
+
+			params: {
+				model: modelId,
+				encoding: 'mp3',
+				bit_rate: 48000,
+			},
+
+			method: 'POST',
+
+			headers: {
+				'Content-Type': 'application/json',
+				'Authorization': `Token ${options.apiKey}`,
+			},
+
+			data: {
+				text,
+			},
+
+			responseType: 'arraybuffer'
+		})
+	} catch (e: any) {
+		const response = e.response
+
+		if (response) {
+			logger.log(`Request failed with status code ${response.status}`)
+
+			if (response.data) {
+				logger.log(`Server responded with:`)
+				logger.log(response.data)
+			}
+		}
+
+		throw e
+	}
+
+	logger.start('Decode synthesized audio')
+	const rawAudio = await FFMpegTranscoder.decodeToChannels(new Uint8Array(response.data))
+
+	logger.end()
+
+	return { rawAudio }
+}
+
+export async function getVoiceList() {
+	return voiceList
+}
+
+export interface DeepgramTTSOptions {
+	apiKey?: string
+}
+
+export const defaultDeepgramTTSOptions = {
+	apiKey: undefined,
+}
+
+export const voiceList: SynthesisVoice[] = [
+	{
+		name: 'Asteria',
+		deepgramModelId: 'aura-asteria-en',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Luna',
+		deepgramModelId: 'aura-luna-en',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Stella',
+		deepgramModelId: 'aura-stella-en',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Athena',
+		deepgramModelId: 'aura-athena-en',
+		languages: ['en-GB', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Hera',
+		deepgramModelId: 'aura-hera-en',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'Orion',
+		deepgramModelId: 'aura-orion-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Arcas',
+		deepgramModelId: 'aura-arcas-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Perseus',
+		deepgramModelId: 'aura-perseus-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Angus',
+		deepgramModelId: 'aura-angus-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Orpheus',
+		deepgramModelId: 'aura-orpheus-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Helios',
+		deepgramModelId: 'aura-helios-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'Zeus',
+		deepgramModelId: 'aura-zeus-en',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+]

package/src/synthesis/ElevenLabsTTS.ts

@@ -4,10 +4,13 @@ import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { decodeBase64 } from '../encodings/Base64.js'
+import { isWordOrSymbolWord, splitToWords } from '../nlp/Segmentation.js'
+import { Timeline } from '../utilities/Timeline.js'
 
 const log = logToStderr
 
-export async function synthesize(text: string, voiceId: string, modelId: string, options: ElevenLabsTTSOptions) {
+export async function synthesize(text: string, voiceId: string, language: string, options: ElevenLabsTTSOptions) {
 	const logger = new Logger()
 	logger.start('Request synthesis from ElevenLabs')
 
@@ -17,7 +20,7 @@ export async function synthesize(text: string, voiceId: string, modelId: string,
 
 	try {
 		response = await request<any>({
-			url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`,
+			url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/with-timestamps`,
 
 			method: 'POST',
 
@@ -26,20 +29,26 @@ export async function synthesize(text: string, voiceId: string, modelId: string,
 				'xi-api-key': options.apiKey,
 			},
 
+			params: {
+				output_format: 'mp3_44100_64',
+			},
+
 			data: {
 				text,
 
-				model_id: modelId,
+				model_id: options.modelId,
 
 				voice_setting: {
 					stability: options.stability,
 					similarity_boost: options.similarityBoost,
 					style: options.style,
-					use_speaker_boost: options.useSpeakerBoost
-				}
+					use_speaker_boost: options.useSpeakerBoost,
+				},
+
+				seed: options.seed
 			},
 
-			responseType: 'arraybuffer'
+			responseType: 'json'
 		})
 	} catch (e: any) {
 		const response = e.response
@@ -57,11 +66,43 @@ export async function synthesize(text: string, voiceId: string, modelId: string,
 	}
 
 	logger.start('Decode synthesized audio')
-	const rawAudio = await FFMpegTranscoder.decodeToChannels(new Uint8Array(response.data))
+	const audioData = decodeBase64(response.data.audio_base64)
+	const rawAudio = await FFMpegTranscoder.decodeToChannels(audioData)
+
+	let timeline: Timeline | undefined
+
+	const characters: string[] = response.data.alignment?.characters
+	const characterStartTimes: number[] = response.data.alignment?.character_start_times_seconds
+	const characterEndTimes: number[] = response.data.alignment?.character_end_times_seconds
+
+	if (characters && characterStartTimes && characterEndTimes) {
+		logger.start('Create timeline from returned character timings')
+
+		const referenceText = characters.join('')
+		const words = (await splitToWords(referenceText, language)).filter(w => isWordOrSymbolWord(w))
+
+		timeline = []
+
+		let offset = 0
+
+		for (const word of words) {
+			const wordStartIndex = referenceText.indexOf(word, offset)
+			const wordEndIndex = wordStartIndex + word.length
+
+			timeline.push({
+				type: 'word',
+				text: word,
+				startTime: characterStartTimes[wordStartIndex],
+				endTime: characterEndTimes[wordEndIndex] ?? characterEndTimes[wordEndIndex - 1]
+			})
+
+			offset = wordEndIndex
+		}
+	}
 
 	logger.end()
 
-	return { rawAudio }
+	return { rawAudio, timeline }
 }
 
 export async function getVoiceList(apiKey: string) {
@@ -78,40 +119,36 @@ export async function getVoiceList(apiKey: string) {
 		responseType: 'json'
 	})
 
-	const elevenlabsVoices: any[] = response.data.voices
+	const elevenLabsVoices: any[] = response.data.voices
 
-	const voices: SynthesisVoice[] = elevenlabsVoices.map(elevenlabsVoice => {
-		const modelId: string = elevenlabsVoice?.high_quality_base_model_ids?.[0] ?? 'eleven_monolingual_v1'
-		const accent: string | undefined = elevenlabsVoice?.labels?.accent
-		const gender: VoiceGender = elevenlabsVoice?.labels?.gender ?? 'unknown'
+	const voices: SynthesisVoice[] = elevenLabsVoices.map(elevenLabsVoice => {
+		const gender: VoiceGender = elevenLabsVoice?.labels?.gender ?? 'unknown'
 
 		const supportedLanguages: string[] = []
 
-		if (accent) {
-			if (accent.startsWith('american')) {
-				supportedLanguages.push('en-US')
-			} else if (accent.startsWith('british')) {
-				supportedLanguages.push('en-GB')
-			} else if (accent === 'irish') {
-				supportedLanguages.push('en-IE')
-			} else if (accent == 'australian') {
-				supportedLanguages.push('en-AU')
-			}
-		}
-
-		if (modelId.includes('multilingual')) {
-			supportedLanguages.push('en', ...supporteMultilingualLanguages)
+		let accent: string | undefined = elevenLabsVoice?.labels?.accent
+		accent = accent?.toLowerCase() ?? ''
+
+		if (accent.startsWith('american')) {
+			supportedLanguages.push('en-US')
+		} else if (accent.startsWith('british')) {
+			supportedLanguages.push('en-GB')
+		} else if (accent === 'irish') {
+			supportedLanguages.push('en-IE')
+		} else if (accent == 'australian') {
+			supportedLanguages.push('en-AU')
 		} else {
 			supportedLanguages.push('en')
 		}
 
+		supportedLanguages.push(...supportedLanguagesInMultilingualModels)
+
 		return {
-			name: elevenlabsVoice.name,
+			name: elevenLabsVoice.name,
 			languages: supportedLanguages,
 			gender,
 
-			elevenLabsVoiceId: elevenlabsVoice.voice_id,
-			elevenLabsModelId: modelId
+			elevenLabsVoiceId: elevenLabsVoice.voice_id,
 		}
 	})
 
@@ -120,18 +157,58 @@ export async function getVoiceList(apiKey: string) {
 
 export interface ElevenLabsTTSOptions {
 	apiKey?: string
+	modelId?: string
+
 	stability?: number
 	similarityBoost?: number
 	style?: number
 	useSpeakerBoost?: boolean
+
+	seed?: number
 }
 
 export const defaultElevenLabsTTSOptions = {
 	apiKey: undefined,
+	modelId: 'eleven_multilingual_v2',
+
 	stability: 0.5,
 	similarityBoost: 0.5,
 	style: 0,
-	useSpeakerBoost: true
+	useSpeakerBoost: true,
+
+	seed: undefined,
 }
 
-export const supporteMultilingualLanguages = ['zh', 'ko', 'nl', 'tr', 'sv', 'id', 'tl', 'ja', 'uk', 'el', 'cs', 'fi', 'ro', 'ru', 'da', 'bg', 'ms', 'sk', 'hr', 'ar', 'ta', 'pl', 'de', 'es', 'fr', 'it', 'hi', 'pt']
+export const supportedLanguagesInMultilingualModels = [
+	'ja',
+	'zh',
+	'de',
+	'hi',
+	'fr',
+	'ko',
+	'pt',
+	'it',
+	'es',
+	'id',
+	'nl',
+	'tr',
+	'fil',
+	'pl',
+	'sv',
+	'bg',
+	'ro',
+	'ar',
+	'cs',
+	'el',
+	'fi',
+	'hr',
+	'ms',
+	'sk',
+	'da',
+	'ta',
+	'uk',
+	'ru',
+	'hu',
+	'no',
+	'vi',
+]

package/src/utilities/ObjectUtilities.ts

@@ -30,7 +30,7 @@ export function deepClone<T>(val: T) {
 }
 
 function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
-	if (val == null || typeof val !== 'object') {
+	if (val === undefined || val === null || typeof val !== 'object') {
 		return val
 	}
 
@@ -57,71 +57,88 @@ function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
 
 			seenObjects.pop()
 
-			return <any>clonedArray
+			return clonedArray as any
 		}
 
 		case '[object ArrayBuffer]': {
 			const clonedArray = new Uint8Array(obj.byteLength)
 			clonedArray.set(new Uint8Array(obj))
-			return <any>clonedArray.buffer
+
+			return clonedArray.buffer as any
 		}
 
 		case '[object Int8Array]': {
 			const clonedArray = new Int8Array(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
 		}
 
 		case '[object Uint8Array]': {
 			const clonedArray = new Uint8Array(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
 		}
 
 		case '[object Uint8ClampedArray]': {
 			const clonedArray = new Uint8ClampedArray(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
 		}
 
 		case '[object Int16Array]': {
 			const clonedArray = new Int16Array(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
 		}
 
 		case '[object Uint16Array]': {
 			const clonedArray = new Uint16Array(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
 		}
 
 		case '[object Int32Array]': {
 			const clonedArray = new Int32Array(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
 		}
 
 		case '[object Uint32Array]': {
 			const clonedArray = new Uint32Array(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
 		}
 
 		case '[object Float32Array]': {
 			const clonedArray = new Float32Array(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
 		}
 
 		case '[object Float64Array]': {
 			const clonedArray = new Float64Array(obj.length)
 			clonedArray.set(obj)
-			return <any>clonedArray
+
+			return clonedArray as any
+		}
+
+		case '[object BigInt64Array]': {
+			const clonedArray = new BigInt64Array(obj.length)
+			clonedArray.set(obj)
+
+			return clonedArray as any
 		}
 
 		case '[object Date]': {
-			return <any>new Date(obj.valueOf())
+			return new Date(obj.valueOf()) as any
 		}
 
 		case '[object RegExp]': {