echogarden 0.11.12 → 0.11.13

package/src/synthesis/MicrosoftEdgeTTS.ts

@@ -0,0 +1,298 @@
+import { request } from "gaxios"
+import WebSocket from "ws"
+
+import { escape } from 'html-escaper'
+
+import splitBuffer from "buffer-split"
+
+import * as AzureCognitiveServicesTTS from "./AzureCognitiveServicesTTS.js"
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
+import { Logger } from "../utilities/Logger.js"
+import { OpenPromise } from "../utilities/OpenPromise.js"
+import { getRandomHexString, logToStderr } from "../utilities/Utilities.js"
+import { getRawAudioDuration } from "../audio/AudioUtilities.js"
+
+const traceEnabled = false
+
+const log: typeof logToStderr = traceEnabled ? logToStderr : () => { }
+
+export async function synthesize(
+	text: string,
+	trustedClientToken: string,
+	voice = "Microsoft Server Speech Text to Speech Voice (en-US, AriaNeural)",
+	ssmlPitchString = "+0Hz",
+	ssmlRateString = "+0%",
+	ssmlVolumeString = "+0%") {
+
+	const logger = new Logger()
+	logger.start("Request synthesis from Microsoft Edge cloud API")
+
+	const { audioData, events } = await requestSynthesis(text, trustedClientToken, voice, ssmlPitchString, ssmlRateString, ssmlVolumeString)
+	logger.end()
+
+	const rawAudio = await FFMpegTranscoder.decodeToChannels(audioData, 24000, 1)
+
+	logger.start("Convert boundary events to timeline")
+	const timeline = AzureCognitiveServicesTTS.boundaryEventsToTimeline(events, getRawAudioDuration(rawAudio))
+	logger.end()
+
+	return { rawAudio, timeline }
+}
+
+type SynthesisRequestResult = { audioData: Buffer, events: any[] }
+
+async function requestSynthesis(
+	text: string,
+	trustedClientToken: string,
+	voice = "Microsoft Server Speech Text to Speech Voice (en-US, AriaNeural)",
+	ssmlPitchString = "+0Hz",
+	ssmlRateString = "+0%",
+	ssmlVolumeString = "+0%") {
+
+	const synthesisOpenPromise = new OpenPromise<SynthesisRequestResult>()
+
+	const requestId = getRandomHexString()
+
+	const webSocket = await initializeWebsocketConnection(trustedClientToken)
+
+	const receivedBinaryMessages: Buffer[] = []
+	const receivedEventMessages: any[] = []
+	const audioChunks: Buffer[] = []
+
+	const onMessage = (messageData: Buffer, isBinary: boolean) => {
+		if (isBinary) {
+			const [header, audioChunk] = parseBinaryMessage(messageData)
+
+			if (header['X-RequestId'] != requestId) {
+				log(`Ignoring binary message to different request Id: ${requestId}`)
+
+				return
+			}
+
+			log("\nReceived binary message:")
+			log(header)
+
+			receivedBinaryMessages.push(messageData)
+			receivedBinaryMessages.push(Buffer.from("\n\n"))
+
+			audioChunks.push(audioChunk)
+		} else {
+			const message = messageData.toString("utf8")
+			const [header, content] = parseTextMessage(message)
+
+			if (header['X-RequestId'] != requestId) {
+				log(`Ignoring text message to different request Id: ${requestId}`)
+				return
+			}
+
+			log("\nReceived text message:")
+			log(header)
+			log(content)
+
+			if (header["Path"] == "turn.start") {
+
+			}
+			else if (header["Path"] == "audio.metadata") {
+				receivedEventMessages.push(content["Metadata"][0])
+			}
+			else if (header["Path"] == "turn.end") {
+				const result: SynthesisRequestResult = { audioData: Buffer.concat(audioChunks), events: receivedEventMessages }
+
+				webSocket.off("message", onMessage)
+				webSocket.off("error", onError)
+				webSocket.off("close", onClose)
+
+				synthesisOpenPromise.resolve(result)
+			}
+		}
+	}
+
+	const onError = (err: Error) => {
+		synthesisOpenPromise.reject(err)
+	}
+
+	const onClose = async (code: number, reason: Buffer) => {
+		log("WebSocket closed.")
+		log(code)
+		log(reason.toString())
+
+		if (reason.length > 0) {
+			synthesisOpenPromise.reject(`Websocket closed with code ${code}, reason: ${reason}`)
+		}
+	}
+
+	webSocket.on("message", onMessage)
+	webSocket.on("error", onError)
+	webSocket.on("close", onClose)
+
+	const requestContentSSML =
+		`<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>` +
+		`<voice  name='${voice}'>` +
+		`<prosody pitch='${ssmlPitchString}' rate='${ssmlRateString}' volume='${ssmlVolumeString}'>` +
+		escape(text) +
+		`</prosody>` +
+		`</voice>` +
+		`</speak>`
+
+	const bodyRequestString =
+		`X-RequestId:${requestId}\r\n` +
+		`Content-Type:application/ssml+xml\r\n` +
+		`X-Timestamp:${getTimestampString()}Z\r\n` + // The added "Z" recreates a Microsoft Edge bug.
+		`Path:ssml\r\n\r\n` +
+		requestContentSSML
+
+	log(bodyRequestString)
+
+	webSocket.send(bodyRequestString)
+
+	return synthesisOpenPromise.promise
+}
+
+let existingWebSocketConnection: WebSocket | undefined = undefined
+
+export async function initializeWebsocketConnection(trustedClientToken: string) {
+	const requestOpenPromise = new OpenPromise<WebSocket>()
+
+	if (existingWebSocketConnection && existingWebSocketConnection.readyState == WebSocket.OPEN) {
+		log(`Existing websocket connection is still open. Reusing it.`)
+
+		requestOpenPromise.resolve(existingWebSocketConnection)
+
+		return requestOpenPromise.promise
+	} else {
+		existingWebSocketConnection = undefined
+	}
+
+	const connectionId = getRandomHexString()
+
+	const requestURL = "wss://speech.platform.bing.com/" +
+		"consumer/speech/synthesize/readaloud/edge/v1" +
+		`?TrustedClientToken=${trustedClientToken}` +
+		"&ConnectionId=" + connectionId
+
+	log(requestURL)
+	log("")
+
+	const webSocket = new WebSocket(requestURL, {
+		headers: {
+			"Pragma": "no-cache",
+			"Cache-Control": "no-cache",
+			"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.66 Safari/537.36 Edg/103.0.1264.44",
+			"Origin": "chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold",
+			"Accept-Encoding": "gzip, deflate, br",
+			"Accept-Language": "en-US,en;q=0.9",
+		}
+	})
+
+	const requestMetadata = {
+		"context": {
+			"synthesis": {
+				"audio": {
+					"metadataoptions": {
+						"sentenceBoundaryEnabled": "false",
+						"wordBoundaryEnabled": "true"
+					},
+					"outputFormat": "webm-24khz-16bit-mono-opus"
+				}
+			}
+		}
+	}
+
+	const onOpen = () => {
+		const configRequestString =
+			`X-Timestamp:${getTimestampString()}\r\n` +
+			`Content-Type:application/json; charset=utf-8\r\n` +
+			`Path:speech.config\r\n\r\n` +
+			`${JSON.stringify(requestMetadata, null, 0)}\r\n`
+
+		log(configRequestString)
+
+		webSocket.send(configRequestString)
+
+		existingWebSocketConnection = webSocket
+
+		webSocket.off("open", onOpen)
+		webSocket.off("close", onClose)
+		webSocket.off("error", onError)
+
+		requestOpenPromise.resolve(webSocket)
+	}
+
+	const onClose = (code: number, reason: Buffer) => {
+		log("WebSocket closed.")
+		log(code)
+		log(reason.toString())
+
+		if (reason.length > 0) {
+			requestOpenPromise.reject(`Websocket closed with code ${code}, reason: ${reason}`)
+		}
+	}
+
+	const onError = (err: Error) => {
+		requestOpenPromise.reject(err)
+	}
+
+	webSocket.on("open", onOpen)
+	webSocket.on("close", onClose)
+	webSocket.on("error", onError)
+
+	return requestOpenPromise.promise
+}
+
+export async function getVoiceList(trustedClientToken: string) {
+	const response = await request<any>({
+		method: "GET",
+
+		url: "https://speech.platform.bing.com/consumer/speech/synthesize/" +
+			`readaloud/voices/list?trustedclienttoken=${trustedClientToken}`,
+
+		headers: {
+			"sec-ch-ua": `" Not;A Brand";v="99", "Microsoft Edge";v="103", "Chromium";v="103"`,
+			"dnt": "1",
+			"sec-ch-ua-mobile": "?0",
+			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.66 Safari/537.36 Edg/103.0.1264.44",
+			"sec-ch-ua-platform": "Windows",
+			"accept": "*/*",
+			"sec-fetch-site": "none",
+			"sec-fetch-mode": "cors",
+			"sec-fetch-dest": "empty",
+			"accept-encoding": "gzip, deflate, br",
+			"accept-language": "en-US,en;q=0.9",
+		},
+
+		responseType: "json"
+	})
+
+	return response.data as any[]
+}
+
+function getTimestampString() {
+	const timestampString = new Date(new Date().toLocaleString("en-US", { timeZone: "UTC" })).toString().replace(/GMT.*/, "GMT+0000 (Coordinated Universal Time)")
+	return timestampString
+}
+
+function parseHeaderString(headerString: string) {
+	const headers: any = {}
+
+	for (const headerLine of headerString.split("\r\n")) {
+		const [key, value] = headerLine.split(":")
+		headers[key] = value
+	}
+
+	return headers
+}
+
+function parseTextMessage(message: string) {
+	const [headerString, content] = message.split("\r\n\r\n")
+
+	return [parseHeaderString(headerString), JSON.parse(content)]
+}
+
+function parseBinaryMessage(message: Buffer) {
+	const audioChunk = splitBuffer(message, Buffer.from("Path:audio\r\n"))[1]
+	const headerBuffer = message.subarray(2, message.length - audioChunk.length)
+	const headerString = headerBuffer.toString("utf8").trim()
+
+	return [parseHeaderString(headerString), audioChunk]
+}
+

package/src/synthesis/SamTTS.ts

@@ -0,0 +1,30 @@
+import { RawAudio } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
+
+// SAM (Software Automatic Mouth) speech synthesizer from 1982 (pure JS port)
+//
+// https://github.com/discordier/sam
+//
+// https://habr-com.translate.goog/ru/post/500764/?_x_tr_sl=auto&_x_tr_tl=en
+
+export async function synthesize(text: string, pitch = 64, speed = 72, mouth = 128, throat = 128) {
+	const logger = new Logger()
+	logger.start("Initialize sam module")
+
+	const { default: SamJs } = await import('sam-js')
+
+	const sam = new SamJs({ pitch, speed, mouth, throat })
+
+	logger.start("Synthesize with sam")
+	const samples: Float32Array = sam.buf32(text) as Float32Array
+
+	if (!samples) {
+		throw new Error("Sam TTS failed")
+	}
+
+	const rawAudio: RawAudio = { audioChannels: [samples], sampleRate: 22050 }
+
+	logger.end()
+
+	return { rawAudio }
+}

package/src/synthesis/SapiTTS.ts

@@ -0,0 +1,222 @@
+import { SynthesisVoice } from "../api/API.js"
+import { decodeToChannels } from "../audio/AudioBufferConversion.js"
+import { RawAudio } from "../audio/AudioUtilities.js"
+import { SampleFormat } from "../codecs/WaveCodec.js"
+import { getShortLanguageCode, lcidToIsoLanguageCode } from "../utilities/Locale.js"
+import { Logger } from "../utilities/Logger.js"
+import { Timeline, TimelineEntry } from "../utilities/Timeline.js"
+import { logToStderr } from "../utilities/Utilities.js"
+
+const log = logToStderr
+
+export function synthesize(text: string, voiceName: string, rate = 0, useSpeechPlatform = false) {
+	return new Promise<{ rawAudio: RawAudio, timeline: Timeline }>(async (resolve, reject) => {
+		const logger = new Logger()
+		logger.start("Initialize winax module")
+
+		const { default: WinAX } = await import("winax")
+
+		logger.start("Create SAPI COM object")
+		const sapiVoice = new global.ActiveXObject(useSpeechPlatform ? "Speech.SPVoice" : "SAPI.SPVoice")
+		sapiVoice.EventInterests = 33790
+
+		logger.start("Get SAPI voice list and select best match")
+
+		if (voiceName) {
+			const voiceObjects = sapiVoice.GetVoices()
+
+			for (let i = 0; i < voiceObjects.Count; i++) {
+				const voiceObject = voiceObjects.Item(i)
+				const candidateVoiceName = voiceObject.GetDescription()
+
+				if (candidateVoiceName == voiceName) {
+					sapiVoice.Voice = voiceObject
+				}
+			}
+		}
+
+		sapiVoice.Rate = rate
+
+		// Create phone converter for language
+		const sapiPhoneConverter = new global.ActiveXObject(useSpeechPlatform ? "Speech.SpPhoneConverter" : "SAPI.SpPhoneConverter")
+		const sapiLanguageCodeHex = sapiVoice.Voice.GetAttribute('Language')
+		const sapiLanguageCode = parseInt(sapiLanguageCodeHex, 16)
+
+		sapiPhoneConverter.LanguageId = sapiLanguageCode
+
+		logger.start("Synthesize with SAPI")
+
+		const sampleRate = 22050
+		const bytesPerSecond = sampleRate * 2
+
+		const sapiOutputStream = new global.ActiveXObject('SAPI.SpMemoryStream')
+		sapiOutputStream.Format.Type = 22 // format code code for SAFT22kHz16BitMono
+		sapiVoice.AudioOutputStream = sapiOutputStream
+
+		const dispatchMessagesInterval = setInterval(() => {
+			WinAX.peekAndDispatchMessages()
+		}, 50)
+
+		const connectionPoints = WinAX.getConnectionPoints(sapiVoice)
+		const connectionPoint = connectionPoints[0]
+
+		const methods = connectionPoint.getMethods()
+
+		const events: Timeline = []
+		let lastWordEvent: TimelineEntry | null = null
+		let lastWordCharPos = -1
+
+		connectionPoint.advise({
+			StartStream: () => {
+			},
+
+			Word: (streamId: number, streamPos: number, charPos: number, length: number) => {
+				if (lastWordCharPos == charPos) {
+					return
+				}
+
+				const wordText = text.substring(charPos, charPos + length)
+				const startTime = streamPos / bytesPerSecond
+
+				const wordEvent = { type: "word", text: wordText, startTime, endTime: -1, timeline: [] } as TimelineEntry
+				events.push(wordEvent)
+
+				lastWordEvent = wordEvent
+				lastWordCharPos = charPos
+			},
+
+			Phoneme: (streamId: number, streamPos: number, duration: number, nextPhoneId: number, feature: number, currentPhoneId: number) => {
+				if (events.length == 0) {
+					return
+				}
+
+				const phoneText = sapiPhoneConverter.IdToPhone(currentPhoneId)
+
+				if (phoneText == "," || phoneText == "_") {
+					return
+				}
+
+				const startTime = streamPos / bytesPerSecond
+				const endTime = startTime + (duration / 1000)
+
+				events.push({ type: "phone", text: phoneText, startTime, endTime })
+			},
+
+			EndStream: (streamId: number, streamPos: number) => {
+				clearInterval(dispatchMessagesInterval)
+
+				const audioData = Buffer.from(sapiOutputStream.GetData())
+				const audioChannels = decodeToChannels(audioData, 1, 16, SampleFormat.PCM)
+
+				WinAX.release(sapiOutputStream)
+				WinAX.release(sapiVoice)
+
+				logger.end()
+
+				resolve({ rawAudio: { audioChannels, sampleRate }, timeline: eventsToTimeline(events, audioChannels[0].length / sampleRate) })
+			}
+		})
+
+		sapiVoice.Speak(text)
+	})
+}
+
+export async function getVoiceList(useSpeechPlatform = false) {
+	const { default: WinAX } = await import("winax")
+
+	const sapiVoice = new global.ActiveXObject(useSpeechPlatform ? "Speech.SPVoice" : "SAPI.SPVoice")
+
+	const voiceObjects = sapiVoice.GetVoices()
+
+	const voices: SynthesisVoice[] = []
+
+	for (let i = 0; i < voiceObjects.Count; i++) {
+		const voiceObject = voiceObjects.Item(i)
+		const voiceName = voiceObject.GetDescription()
+		const voiceGender = voiceObject.GetAttribute("Gender")?.toLowerCase()
+
+		const sapiLanguageCodeHex = voiceObject.GetAttribute('Language')
+		const sapiLanguageCode = parseInt(sapiLanguageCodeHex, 16)
+
+		const languageCodes = await lcidToIsoLanguageCode(sapiLanguageCode)
+
+		if (!languageCodes) {
+			throw new Error(`Couldn't translate SAPI language code ${sapiLanguageCode} to ISO, for voice '${voiceName}'`)
+		}
+
+		const resultLanguageCodes: string[] = []
+
+		for (const languageCode of languageCodes) {
+			if (!resultLanguageCodes.includes(languageCode)) {
+				resultLanguageCodes.push(languageCode)
+			}
+
+			const shortLanguageCode = getShortLanguageCode(languageCode)
+			if (!resultLanguageCodes.includes(shortLanguageCode)) {
+				resultLanguageCodes.push(shortLanguageCode)
+			}
+		}
+
+		voices.push({
+			name: voiceName,
+			languages: resultLanguageCodes,
+			gender: voiceGender || "unknown"
+		})
+	}
+
+	WinAX.release(sapiVoice)
+
+	return voices
+}
+
+export async function AssertSAPIAvailable(testForSpeechPlatform = false) {
+	if (process.platform != "win32") {
+		throw new Error(`SAPI is not available on your platform. SAPI is a Microsoft Windows technology that is only runs on a Windows OS.`)
+	}
+
+	try {
+		const { default: WinAX } = await import("winax")
+	} catch (e) {
+		throw new Error(`winax package, which is required for SAPI support, was not found. You can install it by running 'npm install winax -g'.`)
+	}
+
+	try {
+		const voice = new global.ActiveXObject("SAPI.SPVoice")
+	} catch (e) {
+		throw new Error(`Failed creating a SAPI instance: ${e}`)
+	}
+
+	try {
+		const voice = new global.ActiveXObject("Speech.SPVoice")
+	} catch(e) {
+		throw new Error(`Failed creating an msspeech instance. Please ensure you installed the Microsoft Speech Platform runtime correctly.`)
+	}
+}
+
+function eventsToTimeline(events: Timeline, totalDuration: number): Timeline {
+	const timeline: Timeline = []
+
+	for (const event of events) {
+		if (event.type == "word") {
+			timeline.push(event)
+		} else if (event.type == "phone") {
+			if (timeline.length == 0) {
+				throw new Error("Unexpected: phone event preceded a word event")
+			}
+
+			const lastWordEntry = timeline[timeline.length - 1]
+
+			lastWordEntry.endTime = event.endTime
+
+			const phoneTimeline = lastWordEntry.timeline!
+
+			phoneTimeline.push(event)
+		}
+	}
+
+	if (timeline.length > 0 && timeline[timeline.length - 1].endTime == -1) {
+		timeline[timeline.length - 1].endTime = totalDuration
+	}
+
+	return timeline
+}

package/src/synthesis/StreamlabsPollyTTS.ts

@@ -0,0 +1,114 @@
+import { request } from "gaxios"
+import { SynthesisVoice } from "../api/API.js"
+import { trimAudioEnd } from "../audio/AudioUtilities.js"
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
+import { Phrase, splitToFragments } from "../nlp/Segmentation.js"
+import { Logger } from "../utilities/Logger.js"
+import { concatFloat32Arrays, logToStderr } from "../utilities/Utilities.js"
+import { Timeline } from "../utilities/Timeline.js"
+
+const log = logToStderr
+
+const maxTextLengthPerRequest = 200
+
+export async function synthesizeLongText(text: string, voice: string, languageCode: string, sentenceEndPause = 0.75, segmentEndPause = 1.0) {
+	if (text.length == 0) {
+		throw new Error("Text is empty")
+	}
+
+	const logger = new Logger()
+	logger.start("Prepare and split text")
+
+	const fragments = await splitToFragments(text, maxTextLengthPerRequest, languageCode)
+
+	const audioFragments: Float32Array[] = []
+	let fragmentsSampleRate = 0
+
+	const timeline: Timeline = []
+
+	for (let i = 0; i < fragments.length; i++) {
+		const fragment = fragments[i]
+
+		logger.start(`Request synthesis for text fragment ${i + 1}/${fragments.length} from Streamslabs Polly`)
+		const fragmentMp3Stream = await synthesizeFragment(fragment.text, voice)
+
+		if (fragmentMp3Stream.length == 0) {
+			continue
+		}
+
+		const rawAudio = await FFMpegTranscoder.decodeToChannels(fragmentMp3Stream, 24000, 1)
+		fragmentsSampleRate = rawAudio.sampleRate
+
+		let targetEndingSilenceTime: number
+
+		if (fragment.lastSegment?.isSentenceFinalizer) {
+			targetEndingSilenceTime = sentenceEndPause
+		} else if (fragment.lastSegment instanceof Phrase) {
+			targetEndingSilenceTime = segmentEndPause
+		} else {
+			targetEndingSilenceTime = 0
+		}
+
+		const trimmedAudio = trimAudioEnd(rawAudio.audioChannels[0], targetEndingSilenceTime * rawAudio.sampleRate)
+
+		audioFragments.push(trimmedAudio)
+
+		const startTime = timeline.length == 0 ? 0 : timeline[timeline.length - 1].endTime
+		const endTime = startTime + (trimmedAudio.length / fragmentsSampleRate)
+
+		timeline.push({
+			type: "segment",
+			text: fragment.text,
+			startTime,
+			endTime
+		})
+	}
+
+	const rawAudio = { audioChannels: [concatFloat32Arrays(audioFragments)], sampleRate: fragmentsSampleRate }
+
+	logger.end()
+
+	return {
+		rawAudio,
+		timeline
+	}
+}
+
+export async function synthesizeFragment(text: string, voice: string) {
+	const response = await request<any>({
+		url: `https://streamlabs.com/polly/speak`,
+
+		method: "POST",
+
+		data: {
+			voice,
+			text,
+		},
+
+		responseType: "json"
+	})
+
+	const responseObject = response.data
+	const audioUrl = responseObject.speak_url
+	const audioUrlResponse = await request<ArrayBuffer>({ url: audioUrl, responseType: "arraybuffer" })
+
+	return Buffer.from(audioUrlResponse.data)
+}
+
+export const voiceList: SynthesisVoice[] = [
+	{ name: "Brian", languages: ["en-GB", "en"], gender: "male" },
+	{ name: "Emma", languages: ["en-GB", "en"], gender: "female" },
+	{ name: "Russell", languages: ["en-AU", "en"], gender: "male" },
+	{ name: "Joey", languages: ["en-US", "en"], gender: "male" },
+	{ name: "Matthew", languages: ["en-US", "en"], gender: "male" },
+	{ name: "Joanna", languages: ["en-US", "en"], gender: "female" },
+	{ name: "Kimberly", languages: ["en-US", "en"], gender: "female" },
+	{ name: "Amy", languages: ["en-GB", "en"], gender: "female" },
+	{ name: "Geraint", languages: ["en-GB-WLS", "en-GB", "en"], gender: "male" },
+	{ name: "Nicole", languages: ["en-AU", "en"], gender: "female" },
+	{ name: "Justin", languages: ["en-US", "en"], gender: "male" },
+	{ name: "Ivy", languages: ["en-US", "en"], gender: "female" },
+	{ name: "Kendra", languages: ["en-US", "en"], gender: "female" },
+	{ name: "Salli", languages: ["en-US", "en"], gender: "female" },
+	{ name: "Raveena", languages: ["en-IN", "en"], gender: "female" },
+]