echogarden 2.1.2 → 2.2.1

package/src/nlp/PhoneConversion.ts

@@ -5,7 +5,7 @@ export function ipaPhoneToKirshenbaum(ipaPhone: string) {
 		const convertedChar = ipaToKirshenbaum[char]
 
 		if (convertedChar == undefined) {
-			throw new Error(`Could not convert phone character '${char}' to Kirshenbaum encoding`)
+			throw new Error(`Couldn't convert IPA character '${char}' (part of phone '${ipaPhone}' the Kirshenbaum notation`)
 		}
 
 		result += convertedChar || '_'
@@ -252,257 +252,208 @@ const ipaToTimit: { [p: string]: string[] | undefined } = {
 	'ɾ': ['dx'],
 }
 
-// This is adapted from a lookup table on the eSpeak-ng source code
-const ipaToKirshenbaum: { [p: string]: string | undefined } = {
-    '1': '1',
-    '2': '2',
-    '4': '4',
-    '5': '5',
-    '6': '6',
-    '7': '7',
-    '9': '9',
-    ' ': ' ',
-    '!': '!',
-    '\'': '\'',
-    'ʰ': '#',
-    '$': '$',
-    '%': '%',
-    //'æ': '&',
-	'æ': 'a',
-    'ˈ': '\'',
-    '(': '(',
-    ')': ')',
-    'ɾ': '*',
-    '+': '+',
-    'ˌ': ',',
-    '-': '-',
-    '.': '.',
-    '/': '/',
-    'ɒ': '0',
-    'ɜ': '3',
-    'ɵ': '8',
-    'ː': ':',
-    'ʲ': ';',
-    '<': '<',
-    '=': '=',
-    '>': '>',
-    'ʔ': '?',
-    'ə': '@',
-    'ɑ': 'A',
-    'β': 'B',
-    'ç': 'C',
-    'ð': 'D',
-    'ɛ': 'E',
-    'F': 'F',
-    'ɢ': 'G',
-    'ħ': 'H',
-    'ɪ': 'I',
-    'ɟ': 'J',
-    'K': 'K',
-    'ɫ': 'L',
-    'ɱ': 'M',
-    'ŋ': 'N',
-    'ɔ': 'O',
-    'Φ': 'P',
-    'ɣ': 'Q',
-    'ʀ': 'R',
-    'ʃ': 'S',
-    'θ': 'T',
-    'ʊ': 'U',
-    'ʌ': 'V',
-    'œ': 'W',
-    'χ': 'X',
-    'ø': 'Y',
-    'ʒ': 'Z',
-    '̪': '[',
-    '\\': '\\',
-    ']': ']',
-    '^': '^',
-    '_': '_',
-    '`': '`',
-    'a': 'a',
-    'b': 'b',
-    'c': 'c',
-    'd': 'd',
-    'e': 'e',
-    'f': 'f',
-    'ɡ': 'g',
-    'h': 'h',
-    'i': 'i',
-    'j': 'j',
-    'k': 'k',
-    'l': 'l',
-    'm': 'm',
-    'n': 'n',
-    'o': 'o',
-    'p': 'p',
-    'q': 'q',
-    'r': 'r',
-    's': 's',
-    't': 't',
-    'u': 'u',
-    'v': 'v',
-	'w': 'w',
-    'x': 'x',
-    'y': 'y',
-    'z': 'z',
-    '{': '{',
-    '|': '|',
-    '}': '}',
-    '̃': '~',
-    '': '',
-
-	// Extensions
-	'ɚ': '3',
-	'ɹ': 'r',
+export const ipaToKirshenbaum: Record<string, string> = {
+	//// Vowels
+
+	// Close
+	'i': 'i',
+	'y': 'y',
+	'ɨ': 'i"',
+	'ʉ': 'u"',
+	'ɯ': 'u-',
+	'u': 'u',
+
+	// Near-close
+	'ɪ': 'I',
+	'ɩ': 'I',
+	'ᵻ': 'I',
+	'ʏ': 'I.',
+	'ʊ': 'U',
+	'ɷ': 'U',
+
+	// Close-mid
+	'e': 'e',
+	'ø': 'Y',
+	//'ɘ': '@<umd>', // Reference
+	'ɘ': '@', // Simplified
+	'ɵ': '@.',
+	//'ɵ': '8', // eSpeak
+	'ɤ': 'o-',
+	'o': 'o',
+
+	// Mid:
+	'ə': '@',
+
+	// Open-mid:
+	'ɛ': 'E',
+	'œ': 'W',
+	//'ɜ': '3', // eSpeak
+	'ɜ': 'V"',
+	'ɞ': 'O"',
+	'ʌ': 'V',
+	'ɔ': 'O',
+
+	// Near-open
+	//'æ': '&', // Reference. note: passing '&' to eSpeak in SSML (XML) mode will cause an error due to escaping conflict
+	'æ': 'a', // eSpeak
 	'ɐ': 'a#',
-	'ᵻ': 'i',
-	'̩': ','
-}
 
-/*
-// Source: https://github.com/coruus/ascii-ipa/blob/master/kirshenbaum.py
-// Conversion regex: \('.*?': '.*?'\)
-// Replace: $2: $1
-const ipaToKirshenbaum2: { [p: string]: string | undefined } = {
+	// Open
+	'a': 'a',
+	//'ɶ': '&.', // Reference. note: passing '&' to eSpeak in SSML (XML) mode will cause an error due to escaping conflict
+	'ɶ': 'W#', // eSpeak
+	'ä': 'a"',
+	'ɒ̈': 'A".',
+	'ɑ': 'A',
+	//'ɒ': 'A.', // Reference
+	'ɒ': '0', // eSpeak
+
+	// Rhotic
+	'ɚ': 'R',
+	//'ɚ': '3', // eSpeak
+	//'ɝ': 'R<umd>', // Reference
+	'ɝ': 'R', // Simplified
+
+	// More vowels
+	'ᵊ': '',
+
+	//// Consonants
+
+	// Nasals
 	'm': 'm',
+	'ɱ': 'M',
+	'n̪': 'n[',
+	'n': 'n',
+	'ɳ': 'n.',
+	'ṇ': 'n.',
+	'ɲ': 'n^',
+	'ŋ': 'N',
+	'ɴ': 'n"',
+	//'n͡g': 'n<lbv>',
+
+	// Stops
 	'p': 'p',
 	'b': 'b',
+	't̪': 't[',
+	'd̪': 'd[',
+	't': 't',
+	'd': 'd',
+	'ʈ': 't.',
+	'ṭ': 't.',
+	'ɖ': 'd.',
+	'ḍ': 'd.',
+	'c': 'c',
+	'ɟ': 'J',
+	'k': 'k',
+	'ɡ': 'g',
+	'q': 'q',
+	'ɢ': 'G',
+	//'k͡p': 't<lbv>',
+	//'ɡ͡b': 'd<lbv>',
+	'ʔ': '?',
+
+	// Fricatives
+	'φ': 'P',
 	'Φ': 'P',
 	'β': 'B',
-	'ʙ': 'b<trl>',
-	'pʼ': 'p`',
-	'ɓ': 'b`',
-	'ʘ': 'p!',
-	'ɱ': 'M',
 	'f': 'f',
 	'v': 'v',
-	'ʋ': 'r<lbd>',
-	'n\u032a': 'n[',
-	't\u032a': 't[',
 	'θ': 'T',
 	'ð': 'D',
-	'r\u032a': 'r[',
-	'l\u032a': 'l[',
-	't\u032a\u02bc': 't[`',
-	'ɗ': 'd`',
-	'ʇ': 't!',
-	'n': 'n',
-	't': 't',
-	'd': 'd',
 	's': 's',
 	'z': 'z',
-	'ɬ': 's<lat>',
-	'ɮ': 'z<lat>',
-	'ɹ': 'r',
-	'l': 'l',
-	'ʀ': 'r<trl>',
-	'ɾ': '*',
-	'ɺ': '*<lat>',
-	't\u02bc': 't`',
-	'ʗ': 'c!',
-	'ʖ': 'l!',
-	'ɳ': 'n.',
-	'ʈ': 't.',
-	'ɖ': 'd.',
 	'ʂ': 's.',
+	'ṣ': 's.',
 	'ʐ': 'z.',
-	//'ɖ': 'r.',
-	'ɭ': 'l.',
-	'ɽ': '*.',
+	'ẓ': 'z.',
 	'ʃ': 'S',
 	'ʒ': 'Z',
-	'n^': 'n^',
-	'c': 'c',
-	'ɟ': 'J',
 	'ç': 'C',
 	'ʝ': 'C<vcd>',
-	'j': 'j',
-	'ɥ': 'j<rnd>',
-	'ʎ': 'l^',
-	'ʄ': 'J`',
-	'ŋ': 'N',
-	'k': 'k',
-	'g': 'g',
 	'x': 'x',
 	'ɣ': 'Q',
-	'ɰ': 'j<vel>',
-	'ɫ': 'L',
-	//'ɬ': '{vls,alv,lat,frc}',
-	'k\u02bc': 'k`',
-	'g\u02bc': 'g`',
-	'ʞ': 'k!',
-	'n\u2030g': 'n<lbv>',
-	'k\u2030p': 't<lbv>',
-	'g\u2030b': 'n<lbv>',
+	'χ': 'X',
+	'ʁ': 'g"', // Reference
+	//'ʁ': 'r', // eSpeak
 	'ʍ': 'w<vls>',
 	'w': 'w',
-	'ɴ': 'n'',
-	'q': 'q',
-	'ɢ': 'G',
-	'χ': 'X',
-	'ʁ': 'g'',
-	//'ʀ': 'r'',
-	'ʠ': 'q`',
-	'ʛ': 'G`',
 	'ħ': 'H',
 	'ʕ': 'H<vcd>',
-	'ʔ': '?',
 	'h': 'h',
 	'ɦ': 'h<?>',
-	'i': 'i',
-	'y': 'y',
-	'ɪ': 'I',
-	'ʏ': 'I.',
-	'e': 'e',
-	'ø': 'Y',
-	'ɛ': 'E',
-	'œ': 'W',
-	'æ': '&',
-	'ɶ': '&.',
-	'ɨ': 'i'',
-	'ʉ': 'u'',
-	'ɘ': '@<umd>',
-	'ɝ': 'R<umd>',
-	'ə': '@',
-	'ɚ': 'R',
-	'ɵ': '@.',
-	//'ɜ': 'V'',
-	'ɜ': '3',
-	'ɞ': 'O'',
-	'a': 'a',
-	'ɯ': 'u-',
-	'u': 'u',
-	'ʊ': 'U',
-	'ɤ': 'o-',
-	'o': 'o',
-	'ʌ': 'V',
-	'ɔ': 'O',
-	'ɑ': 'A',
-	'ɒ': 'A.',
 
-	'ː': ':',
-	'\u0322': '.',
-	'\u02bc': '`',
-	'\u032a': '[',
-	'\u02b2': ';',
-	''': ''',
-	'^': '^',
-	'\u0334': '<H>',
-	'\u02b0': '<h>',
-	'\u02da': '<unx>',
-	'\u0325': '<vls>',
-	//'\u02da': '<o>',
-	'\u02b3': '<r>',
-	'\u02b7': '<w>',
-	'\u02b1': '<?>',
-
-	'\u0303': '~',
-	//'\u0334': '~'
-
-	'ˈ': ''',
+	// Approximants
+	'ʋ': 'r<lbd>',
+	'ɹ̪': 'r[',
+	'ɹ': 'r',
+	'ɻ': 'r.',
+	'ɻ̣': 'r.',
+	'j': 'j',
+	'ɥ': 'j<rnd>',
+	'ɰ': 'j<vel>',
+
+	// Laterals
+	'l̪': 'l[',
+	'l': 'l',
+	'ɭ': 'l.',
+	'ḷ': 'l.',
+	'ʎ': 'l^',
+	'ʟ': 'L',
+
+	// Trills
+	'ʙ': 'b<trl>',
+	//'r': 'r<trl>', // Reference alv trl
+	'r': 'R', // eSpeak
+	'ʀ': 'r"', // Reference uvl trl
+
+	// Flaps
+	'ɾ': '*',
+	'ɽ': '*.',
+	'ṛ': '*.',
+
+	// Ejectives
+	'pʼ': 'p`',
+	'tʼ': 't`',
+	'cʼ': 'c`',
+	'kʼ': 'k`',
+	'qʼ': 'q`',
+	'ʠ': 'q`',
+
+	// Implosives
+	'ɓ': 'b`',
+	'ɗ': 'd`',
+	'ʄ': 'J`',
+	'ɠ': 'g`',
+	'ʛ': 'G`',
+
+	// Clicks
+	'ʘ': 'p!',
+	'ǀ': 't!',
+	'ʇ': 't!',
+	'ǃ': 'c!',
+	'ǂ': 'c!',
+	'ʗ': 'c!',
+	'ʞ': 'k!',
+	'ǁ': 'l!',
+	'ʖ': 'l!',
+
+	// Lateral fricative
+	'ɬ': 'L',
+	'ɮ': 'z<lat>',
+	'ɫ': 'L',
+
+	// Lateral flap
+	'ɺ': '*<lat>',
+
+	// Other consonants
+	'ʰ': '#',
+	'ʲ': ';',
+	'̪': '[',
+	'̃': '~',
+
+	// Stress marks and other misc characters
+	'ˈ': '\'',
 	'ˌ': ',',
-	' ': ' ',
-	'\n': '\n'
+	'ː': ':',
+	'-': '-',
 }
-*/

package/src/recognition/DeepgramSTT.ts

@@ -0,0 +1,136 @@
+import { request } from 'gaxios'
+import { RawAudio } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+
+export async function recognize(rawAudio: RawAudio, languageCode: string | undefined, options: DeepgramSTTOptions) {
+	const logger = new Logger()
+
+	logger.start('Initialize Deepgram recognition')
+
+	options = extendDeep(defaultDeepgramSTTOptions, options)
+
+	if (!options.apiKey) {
+		throw new Error('No Deepgram API key provided')
+	}
+
+	// Prepare API request
+	const params: Record<string, string> = {
+		model: options.model || 'whisper-large',
+		encoding: 'flac',
+		//punctuate: 'true', // Problem when enabled: words in timeline are not capitalized, causing errors
+	}
+
+	// Set language or enable auto-detection
+	if (languageCode) {
+		params.language = languageCode
+	} else {
+		params.detect_language = 'true'
+	}
+
+	// Set audio encoding parameters
+	logger.start('Convert audio to FLAC format')
+
+	const audioData = await FFMpegTranscoder.encodeFromChannels(
+		rawAudio,
+		FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('flac')
+	)
+
+	logger.start('Send request to Deepgram API')
+
+	const response = await request<any>({
+		method: 'POST',
+
+		url: 'https://api.deepgram.com/v1/listen',
+
+		params,
+
+		headers: {
+			'Authorization': `Token ${options.apiKey}`,
+			'Content-Type': 'audio/flac',
+			'Accept': 'application/json'
+		},
+
+		body: audioData,
+
+		responseType: 'json',
+	})
+
+	const deepgramResponse: DeepgramResponse = response.data
+
+	const firstAlternative = deepgramResponse.results?.channels[0]?.alternatives[0]
+
+	// Extract transcript and create timeline
+	const transcript = firstAlternative?.transcript || ''
+
+	let timeline: Timeline = []
+
+	// Extract word-level timing information if available
+	const words = firstAlternative?.words || []
+
+	if (words.length > 0) {
+		timeline = words.map((word: DeepgramWordEntry): TimelineEntry => ({
+			type: 'word',
+			text: word.word,
+			startTime: word.start,
+			endTime: word.end,
+			confidence: word.confidence,
+		}))
+	}
+
+	logger.end()
+
+	return { transcript, timeline }
+}
+
+export interface DeepgramSTTOptions {
+	apiKey?: string
+	model?: string
+}
+
+export const defaultDeepgramSTTOptions: DeepgramSTTOptions = {
+	apiKey: undefined,
+	model: 'nova-2'
+}
+
+interface DeepgramWordEntry {
+	word: string
+	start: number
+	end: number
+	confidence: number
+}
+
+interface DeepgramAlternative {
+	transcript: string
+	confidence: number
+	words: DeepgramWordEntry[]
+}
+
+interface DeepgramChannel {
+	alternatives: DeepgramAlternative[]
+}
+
+interface DeepgramResponse {
+	metadata?: {
+		transaction_key: string
+		request_id: string
+		sha256: string
+		created: string
+		duration: number
+		channels: number
+		models: string[]
+
+		model_info?: {
+			name: string
+			version: string
+		}
+	}
+
+	results?: {
+		channels: DeepgramChannel[]
+
+		detected_language?: string
+	}
+}

package/src/recognition/GoogleCloudSTT.ts

@@ -50,7 +50,6 @@ export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode
 
 		data: requestBody,
 
-
 		responseType: 'json'
 	})
 

package/src/synthesis/{ElevenlabsTTS.ts → ElevenLabsTTS.ts}

@@ -7,11 +7,11 @@ import { extendDeep } from '../utilities/ObjectUtilities.js'
 
 const log = logToStderr
 
-export async function synthesize(text: string, voiceId: string, modelId: string, options: ElevenlabsTTSOptions) {
+export async function synthesize(text: string, voiceId: string, modelId: string, options: ElevenLabsTTSOptions) {
 	const logger = new Logger()
 	logger.start('Request synthesis from ElevenLabs')
 
-	options = extendDeep(defaultElevenlabsTTSOptions, options)
+	options = extendDeep(defaultElevenLabsTTSOptions, options)
 
 	let response: GaxiosResponse<any>
 
@@ -118,7 +118,7 @@ export async function getVoiceList(apiKey: string) {
 	return voices
 }
 
-export interface ElevenlabsTTSOptions {
+export interface ElevenLabsTTSOptions {
 	apiKey?: string
 	stability?: number
 	similarityBoost?: number
@@ -126,7 +126,7 @@ export interface ElevenlabsTTSOptions {
 	useSpeakerBoost?: boolean
 }
 
-export const defaultElevenlabsTTSOptions = {
+export const defaultElevenLabsTTSOptions = {
 	apiKey: undefined,
 	stability: 0.5,
 	similarityBoost: 0.5,

package/src/synthesis/EspeakTTS.ts

@@ -169,9 +169,12 @@ export async function synthesizeFragments(fragments: string[], espeakOptions: Es
 
 		fragment = simplifyPunctuationCharacters(fragment)
 
-		fragment = fragment
-			.replaceAll('<', '&lt;')
-			.replaceAll('>', '&gt;')
+		//fragment = encodeHTMLAngleBrackets(fragment)
+		fragment = fragment.replaceAll('<', '_').replaceAll('>', '_')
+
+		if (fragment.split('').every(c => c === ':')) {
+			fragment = ','
+		}
 
 		if (espeakOptions.insertSeparators && canInsertSeparators) {
 			const separator = ` | `
@@ -597,9 +600,9 @@ export const defaultEspeakOptions: EspeakOptions = {
 	insertSeparators: false
 }
 
-export async function testKirshenbaumPhonemization(text: string) {
-	const ipaPhonemizedSentence = (await phonemizeSentence(text, 'en-us')).flatMap(clause => clause)
-	const kirshenbaumPhonemizedSentence = (await phonemizeSentence(text, 'en-us', undefined, false)).flatMap(clause => clause)
+export async function testKirshenbaumPhonemization(text: string, language = 'en-us') {
+	const ipaPhonemizedSentence = (await phonemizeSentence(text, language)).flatMap(clause => clause)
+	const kirshenbaumPhonemizedSentence = (await phonemizeSentence(text, language, undefined, false)).flatMap(clause => clause)
 
 	const ipaFragments = ipaPhonemizedSentence.map(word => word.join(''))
 
@@ -610,6 +613,8 @@ export async function testKirshenbaumPhonemization(text: string) {
 			ipaPhoneToKirshenbaum(phoneme)).join(''))
 
 	for (let i = 0; i < fragments.length; i++) {
-		log(`IPA: ${ipaFragments[i]} | converted: ${fragments[i]} | ground truth: ${kirshenbaumFragments[i]}`)
+		if (fragments[i] !== kirshenbaumFragments[i]) {
+			log(`IPA: ${ipaFragments[i]} | converted: ${fragments[i]} | ground truth: ${kirshenbaumFragments[i]}`)
+		}
 	}
 }