echogarden 0.11.12 → 0.11.13

package/src/synthesis/FliteTTS.ts

@@ -0,0 +1,387 @@
+import { SynthesisVoice } from "../api/API.js"
+import { decodeWaveBuffer } from "../audio/AudioUtilities.js"
+import { Logger } from "../utilities/Logger.js"
+import { getRandomHexString, logToStderr, resolveModuleMainPath } from "../utilities/Utilities.js"
+import { getAppTempDir, readFile, open, close, remove, ensureDir } from "../utilities/FileSystem.js"
+import path from "node:path"
+import { escape } from 'html-escaper'
+
+const log = logToStderr
+
+export type FliteVoiceName = "kal" | "kal16" | "awb" | "rms" | "slt" | string
+
+let fliteModuleObject: WebAssembly.Module
+
+export async function synthesize(text: string, voice: FliteVoiceName, voiceDir: string | undefined, rate: number, pitchMeanHz?: number, pitchStdDev?: number) {
+	const logger = new Logger()
+	logger.start("Get Flite WASI instance")
+
+	const randomId = getRandomHexString(16)
+
+	const outFileName = `${randomId}.out.wav`
+	const stdOutFileName = `${randomId}.stdout`
+
+	const tempDir = getAppTempDir("flite")
+	await ensureDir(tempDir)
+
+	const tempAudioFilePath = path.join(tempDir, outFileName)
+	const tempStdOutFilePath = path.join(tempDir, stdOutFileName)
+
+	const stdOutFileFd = await open(tempStdOutFilePath, "w+")
+
+	if (voiceDir) {
+		voice = `voices/${voice}.flitevox`
+	}
+
+	const optionArgs = [
+		'-voice', voice,
+		'--setf', `duration_stretch=${1.0 / rate}`,
+	]
+
+	if (pitchMeanHz != null) {
+		optionArgs.push('--setf', `int_f0_target_mean=${pitchMeanHz}`)
+	}
+
+	if (pitchStdDev != null) {
+		optionArgs.push('--setf', `int_f0_target_stddev=${pitchStdDev}`)
+	}
+
+	const preopens: { [k: string]: string } = {
+		'.': tempDir,
+	}
+
+	if (voiceDir != undefined) {
+		preopens['./voices'] = voiceDir
+	}
+
+	const { WASI } = await import('wasi')
+
+	const wasi = new WASI({
+		env: {
+		},
+
+		args: ['--',
+			//'-ssml',
+			'-psdur',
+			...optionArgs,
+			//` ${escape(text)} `,
+			` ${text} `,
+			outFileName
+		],
+
+		preopens,
+
+		stdout: stdOutFileFd,
+
+		returnOnExit: true
+	})
+
+	// Some WASI binaries require `const importObject = { wasi_unstable: wasi.wasiImport }`
+	const importObject = { wasi_snapshot_preview1: wasi.wasiImport }
+
+	const moduleObject = await getModuleObject()
+	const instance = await WebAssembly.instantiate(moduleObject, importObject)
+
+	logger.start("Synthesize with flite")
+	const exitCode: number = wasi.start(instance) as any
+
+	await close(stdOutFileFd)
+
+	async function cleanup() {
+		await remove(tempAudioFilePath)
+		await remove(tempStdOutFilePath)
+	}
+
+	if (exitCode != 0) {
+		await cleanup()
+		throw new Error(`Flite failed with exit code ${exitCode}`)
+	}
+
+	const waveData = await readFile(tempAudioFilePath)
+	const stdOutString = await readFile(tempStdOutFilePath, "utf8")
+
+	await cleanup()
+
+	const events = parseEventsFromTrace(stdOutString)
+
+	const { rawAudio } = decodeWaveBuffer(waveData)
+
+	logger.end()
+
+	return { rawAudio, events }
+}
+
+async function getModuleObject() {
+	if (!fliteModuleObject) {
+		const fliteWasiPath = await resolveModuleMainPath('@echogarden/flite-wasi')
+
+		fliteModuleObject = await WebAssembly.compile(await readFile(fliteWasiPath))
+	}
+
+	return fliteModuleObject
+}
+
+function parseEventsFromTrace(eventTrace: string) {
+	//log(eventTrace)
+
+	const eventStrings = eventTrace.trim().split(" ")
+	const eventCount = eventStrings.length
+
+	const events: FliteEvent[] = []
+	let phraseStartOffset = 0
+
+	for (let i = 0; i < eventCount; i++) {
+		const eventString = eventStrings[i]
+		const splitPoint = eventString.lastIndexOf(":")
+
+		const id = eventString.substring(0, splitPoint)
+		const startTime = events.length > 0 ? events[events.length - 1].endTime : 0
+
+		let eventType: FliteEventType
+
+		if (id == "pau") {
+			eventType = "pause"
+		} else if (id == "\npau") {
+			eventType = "phrasePause"
+
+			phraseStartOffset = startTime
+		} else {
+			eventType = "phone"
+		}
+
+		const endTime = phraseStartOffset + parseFloat(eventString.substring(splitPoint + 1))
+
+		events.push({
+			type: eventType,
+			id,
+			startTime,
+			endTime
+		})
+	}
+
+	return events
+}
+
+export type FliteEventType = "phone" | "pause" | "phrasePause"
+
+export type FliteEvent = {
+	type: FliteEventType,
+	id: string,
+	startTime: number,
+	endTime: number
+}
+
+export const voiceList: SynthesisVoice[] = [
+	// Built-in voices
+	{
+		name: "slt",
+		languages: ["en-US", "en"],
+		gender: "female"
+	},
+	{
+		name: "kal16",
+		languages: ["en-US", "en"],
+		gender: "male"
+	},
+	{
+		name: "rms",
+		languages: ["en-US", "en"],
+		gender: "male"
+	},
+	{
+		name: "awb",
+		languages: ["en-GB-SCOTLAND", "en-GB", "en"],
+		gender: "male"
+	},
+	{
+		name: "kal",
+		languages: ["en-US", "en"],
+		gender: "male"
+	},
+
+
+	// Voices loaded from packages
+	// US English voices
+	{
+		name: "cmu_us_aew",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_aew"
+	},
+	{
+		name: "cmu_us_ahw",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_ahw"
+	},
+	{
+		name: "cmu_us_aup",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_aup"
+	},
+	{
+		name: "cmu_us_awb",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_awb"
+	},
+	{
+		name: "cmu_us_axb",
+		languages: ["en-US", "en"],
+		gender: "female",
+		packageName: "flite-cmu_us_axb"
+	},
+	{
+		name: "cmu_us_bdl",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_bdl"
+	},
+	{
+		name: "cmu_us_clb",
+		languages: ["en-US", "en"],
+		gender: "female",
+		packageName: "flite-cmu_us_clb"
+	},
+	{
+		name: "cmu_us_eey",
+		languages: ["en-US", "en"],
+		gender: "female",
+		packageName: "flite-cmu_us_eey"
+	},
+	{
+		name: "cmu_us_fem",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_fem"
+	},
+	{
+		name: "cmu_us_gka",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_gka"
+	},
+	{
+		name: "cmu_us_jmk",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_jmk"
+	},
+	{
+		name: "cmu_us_ksp",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_ksp"
+	},
+	{
+		name: "cmu_us_ljm",
+		languages: ["en-US", "en"],
+		gender: "female",
+		packageName: "flite-cmu_us_ljm"
+	},
+	{
+		name: "cmu_us_lnh",
+		languages: ["en-US", "en"],
+		gender: "female",
+		packageName: "flite-cmu_us_lnh"
+	},
+	{
+		name: "cmu_us_rms",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_rms"
+	},
+	{
+		name: "cmu_us_rxr",
+		languages: ["en-US", "en"],
+		gender: "male",
+		packageName: "flite-cmu_us_rxr"
+	},
+	{
+		name: "cmu_us_slp",
+		languages: ["en-US", "en"],
+		gender: "female",
+		packageName: "flite-cmu_us_slp"
+	},
+	{
+		name: "cmu_us_slt",
+		languages: ["en-US", "en"],
+		gender: "female",
+		packageName: "flite-cmu_us_slt"
+	},
+
+	// Indic voices
+	{
+		name: "cmu_indic_ben_rm",
+		languages: ["be", "bn"],// Bengali
+		gender: "female",
+		packageName: "flite-cmu_indic_ben_rm"
+	},
+	{
+		name: "cmu_indic_guj_ad",
+		languages: ["gu"],// Gujarati
+		gender: "male",
+		packageName: "flite-cmu_indic_guj_ad"
+	},
+	{
+		name: "cmu_indic_guj_dp",
+		languages: ["gu"],// Gujarati
+		gender: "female",
+		packageName: "flite-cmu_indic_guj_dp"
+	},
+	{
+		name: "cmu_indic_guj_kt",
+		languages: ["gu"],// Gujarati
+		gender: "female",
+		packageName: "flite-cmu_indic_guj_kt"
+	},
+	{
+		name: "cmu_indic_hin_ab",
+		languages: ["hi"],// Hindi
+		gender: "female",
+		packageName: "flite-cmu_indic_hin_ab"
+	},
+	{
+		name: "cmu_indic_kan_plv",
+		languages: ["ka"],// Kannada
+		gender: "female",
+		packageName: "flite-cmu_indic_kan_plv"
+	},
+	{
+		name: "cmu_indic_mar_aup",
+		languages: ["mr"],// Marathi
+		gender: "male",
+		packageName: "flite-cmu_indic_mar_aup"
+	},
+	{
+		name: "cmu_indic_mar_slp",
+		languages: ["mr"],// Marathi
+		gender: "female",
+		packageName: "flite-cmu_indic_mar_slp"
+	},
+	{
+		name: "cmu_indic_pan_amp",
+		languages: ["pa"],// Punjabi
+		gender: "female",
+		packageName: "flite-cmu_indic_pan_amp"
+	},
+	{
+		name: "cmu_indic_tel_kpn",
+		languages: ["te"],// Telugu
+		gender: "female",
+		packageName: "flite-cmu_indic_tel_kpn"
+	},
+	{
+		name: "cmu_indic_tel_sk",
+		languages: ["te"],// Telugu
+		gender: "male",
+		packageName: "flite-cmu_indic_tel_sk"
+	},
+	{
+		name: "cmu_indic_tel_ss",
+		languages: ["te"],// Telugu
+		gender: "female",
+		packageName: "flite-cmu_indic_tel_ss"
+	},
+]

package/src/synthesis/GoogleCloudTTS.ts

@@ -0,0 +1,112 @@
+import { request } from "gaxios"
+import { Logger } from "../utilities/Logger.js"
+import { logToStderr } from "../utilities/Utilities.js"
+
+const log = logToStderr
+
+export async function synthesize(text: string, apiKey: string, languageCode = "en-US", voice = "en-US-Wavenet-C", speakingRate = 1.0, pitchDeltaSemitones = 0.0, volumeGainDb = 0.0, ssml = false, audioEncoding: AudioEncoding = "MP3_64_KBPS", sampleRate = 24000) {
+	const logger = new Logger()
+	logger.start("Request synthesis from Google Cloud")
+
+	const requestBody = {
+		input: {
+			text: undefined as (string | undefined),
+			ssml: undefined as (string | undefined)
+		},
+
+		voice: {
+			languageCode,
+			name: voice
+		},
+
+		audioConfig: {
+			audioEncoding,
+			speakingRate,
+			pitch: pitchDeltaSemitones,
+			volumeGainDb,
+			sampleRateHertz: sampleRate
+		},
+
+		enableTimePointing: ["SSML_MARK"]
+	}
+
+	if (ssml) {
+		requestBody.input.ssml = text
+	} else {
+		requestBody.input.text = text
+	}
+
+	const response = await request<any>({
+		method: "POST",
+
+		url: `https://texttospeech.googleapis.com/v1beta1/text:synthesize`,
+
+		params: {
+			"key": apiKey
+		},
+
+		headers: {
+			"User-Agent": ""
+		},
+
+		data: requestBody,
+
+		responseType: "json"
+	})
+
+	logger.start("Parse result")
+
+	const result = parseResponseBody(response.data)
+
+	logger.end()
+
+	return result
+}
+
+function parseResponseBody(responseBody: any) {
+	const audioData = Buffer.from(responseBody.audioContent, "base64")
+	const timepoints: timePoint[] = responseBody.timepoints
+
+	return { audioData, timepoints }
+}
+
+// Voices with audio samples: https://cloud.google.com/text-to-speech/docs/voices
+export async function getVoiceList(apiKey: string) {
+	const requestURL = `https://texttospeech.googleapis.com/v1beta1/voices`
+
+	const response = await request<any>({
+		method: "GET",
+
+		url: requestURL,
+
+		params: {
+			"key": apiKey
+		},
+
+		headers: {
+			"User-Agent": ""
+		},
+
+		responseType: "json"
+	})
+
+	const responseData = response.data
+
+	const voices: GoogleCloudVoice[] = responseData.voices
+
+	return voices
+}
+
+export type GoogleCloudVoice = {
+	name: string
+	languageCodes: string[]
+	ssmlGender: "MALE" | "FEMALE"
+	naturalSampleRateHertz: number
+}
+
+export type AudioEncoding = "LINEAR16" | "MP3" | "MP3_64_KBPS" | "OGG_OPUS" | "MULAW" | "ALAW"
+
+export type timePoint = {
+	markName: string,
+	timeSeconds: number
+}

package/src/synthesis/GoogleTranslateTTS.ts

@@ -0,0 +1,210 @@
+import { request } from "gaxios"
+import { Phrase, splitToFragments } from "../nlp/Segmentation.js"
+import { concatFloat32Arrays, logToStderr } from "../utilities/Utilities.js"
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
+import { trimAudioEnd } from "../audio/AudioUtilities.js"
+import { Logger } from "../utilities/Logger.js"
+import { Timeline } from "../utilities/Timeline.js"
+import { getShortLanguageCode } from "../utilities/Locale.js"
+
+const log = logToStderr
+
+const maxTextLengthPerRequest = 200
+
+export async function synthesizeLongText(text: string, languageCode = "en", tld = "us", sentenceEndPause = 0.75, segmentEndPause = 1.0) {
+	if (text.length == 0) {
+		throw new Error("Text is empty")
+	}
+
+	const logger = new Logger()
+	logger.start("Prepare and split text")
+
+	const fragments = await splitToFragments(text, maxTextLengthPerRequest, languageCode)
+
+	const audioFragments: Float32Array[] = []
+	let fragmentsSampleRate = 0
+
+	const timeline: Timeline = []
+
+	for (let i = 0; i < fragments.length; i++) {
+		const fragment = fragments[i]
+
+		logger.start(`Request synthesis for text fragment ${i + 1}/${fragments.length} from Google Translate`)
+		const fragmentMp3Stream = await synthesizeShortText(fragment.text, languageCode, tld)
+
+		if (fragmentMp3Stream.length == 0) {
+			continue
+		}
+
+		const rawAudio = await FFMpegTranscoder.decodeToChannels(fragmentMp3Stream, 24000, 1)
+		fragmentsSampleRate = rawAudio.sampleRate
+
+		let targetEndingSilenceTime: number
+
+		if (fragment.lastSegment?.isSentenceFinalizer) {
+			targetEndingSilenceTime = 0.75
+		} else if (fragment.lastSegment instanceof Phrase) {
+			targetEndingSilenceTime = 0.1
+		} else {
+			targetEndingSilenceTime = 0
+		}
+
+		const trimmedAudio = trimAudioEnd(rawAudio.audioChannels[0], targetEndingSilenceTime * rawAudio.sampleRate)
+
+		audioFragments.push(trimmedAudio)
+
+		const startTime = timeline.length == 0 ? 0 : timeline[timeline.length - 1].endTime
+		const endTime = startTime + (trimmedAudio.length / fragmentsSampleRate)
+
+		timeline.push({
+			type: "segment",
+			text: fragment.text,
+			startTime,
+			endTime
+		})
+	}
+
+	logger.end()
+
+	const rawAudio = { audioChannels: [concatFloat32Arrays(audioFragments)], sampleRate: fragmentsSampleRate }
+
+	return {
+		rawAudio,
+		timeline
+	}
+}
+
+export async function synthesizeShortText(text: string, languageCode = "en", tld = "us") {
+	if (text.length > maxTextLengthPerRequest) {
+		throw new Error(`Text is ${text.length} characters, which is longer than the maximum of ${maxTextLengthPerRequest}`)
+	}
+
+	if (languageCode != "zh-CN" && languageCode != "zh-TW") {
+		languageCode = getShortLanguageCode(languageCode)
+	}
+
+	const bodyFormParameters = [text, languageCode, null, "null"]
+	const requestForm = [[["jQ1olc", JSON.stringify(bodyFormParameters), null, "generic"]]]
+	const stringifiedForm = JSON.stringify(requestForm)
+	const requestBody = `f.req=${encodeURIComponent(stringifiedForm)}&`
+
+	const response = await request<string>({
+		method: "POST",
+
+		url: `https://translate.google.${tld}/_/TranslateWebserverUi/data/batchexecute`,
+
+		params: {
+			"rpcids": "jQ1olc"
+		},
+
+		headers: {
+			"sec-ch-ua": `".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"`,
+			"x-same-domain": "1",
+			"dnt": "1",
+			"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
+			"sec-ch-ua-mobile": "?0",
+			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
+			"sec-ch-ua-arch": "x86",
+			"sec-ch-ua-full-version": "103.0.5060.114",
+			"sec-ch-ua-platform-version": "10.0.0",
+			"sec-ch-ua-full-version-list": `".Not/A)Brand";v="99.0.0.0", "Google Chrome";v="103.0.5060.114", "Chromium";v="103.0.5060.114"`,
+			"sec-ch-ua-bitness": "64",
+			"sec-ch-ua-model": "",
+			"sec-ch-ua-platform": "Windows",
+			"accept": "*/*",
+			"origin": "https://translate.google.com",
+			"sec-fetch-site": "same-origin",
+			"sec-fetch-mode": "cors",
+			"sec-fetch-dest": "empty",
+			"referer": "https://translate.google.com/",
+			"accept-encoding": "gzip, deflate, br",
+			"accept-language": "en-US,en;q=0.9",
+		},
+
+		body: requestBody,
+
+		responseType: "text"
+	})
+
+	//log(response.data)
+
+	const audioChunks: Buffer[] = []
+
+	for (const line of response.data.split(/\r?\n/)) {
+		const match = line.match(/"jQ1olc","\[\\"(.*)\\"]/)
+
+		if (match != null) {
+			const base64AudioChunk = match[1]
+			audioChunks.push(Buffer.from(base64AudioChunk, "base64"))
+		}
+	}
+
+	const resultMp3Stream = Buffer.concat(audioChunks)
+	return resultMp3Stream
+}
+
+export const supportedLanguageLookup: { [langCode: string]: string } = {
+	"af": "Afrikaans",
+	"ar": "Arabic",
+	"bg": "Bulgarian",
+	"bn": "Bengali",
+	"bs": "Bosnian",
+	"ca": "Catalan",
+	"cs": "Czech",
+	"cy": "Welsh",
+	"da": "Danish",
+	"de": "German",
+	"el": "Greek",
+	"en": "English",
+	"eo": "Esperanto",
+	"es": "Spanish",
+	"et": "Estonian",
+	"fi": "Finnish",
+	"fr": "French",
+	"gu": "Gujarati",
+	"hi": "Hindi",
+	"hr": "Croatian",
+	"hu": "Hungarian",
+	"hy": "Armenian",
+	"id": "Indonesian",
+	"is": "Icelandic",
+	"it": "Italian",
+	"iw": "Hebrew",
+	"ja": "Japanese",
+	"jw": "Javanese",
+	"km": "Khmer",
+	"kn": "Kannada",
+	"ko": "Korean",
+	"la": "Latin",
+	"lv": "Latvian",
+	"mk": "Macedonian",
+	"ms": "Malay",
+	"ml": "Malayalam",
+	"mr": "Marathi",
+	"my": "Myanmar (Burmese)",
+	"ne": "Nepali",
+	"nl": "Dutch",
+	"no": "Norwegian",
+	"pl": "Polish",
+	"pt": "Portuguese",
+	"ro": "Romanian",
+	"ru": "Russian",
+	"si": "Sinhala",
+	"sk": "Slovak",
+	"sq": "Albanian",
+	"sr": "Serbian",
+	"su": "Sundanese",
+	"sv": "Swedish",
+	"sw": "Swahili",
+	"ta": "Tamil",
+	"te": "Telugu",
+	"th": "Thai",
+	"tl": "Filipino",
+	"tr": "Turkish",
+	"uk": "Ukrainian",
+	"ur": "Urdu",
+	"vi": "Vietnamese",
+	"zh-CN": "Chinese",
+	"zh-TW": "Chinese (Mandarin/Taiwan)",
+	"zh": "Chinese (Mandarin)",
+}