echogarden 1.8.7 → 2.0.1

package/src/synthesis/EspeakTTS.ts

@@ -3,7 +3,6 @@ import { int16PcmToFloat32 } from '../audio/AudioBufferConversion.js'
 import { Logger } from '../utilities/Logger.js'
 import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js'
 import { RawAudio, getEmptyRawAudio } from '../audio/AudioUtilities.js'
-import { playAudioWithTimelinePhones } from '../audio/AudioPlayer.js'
 import { getNormalizedFragmentsForSpeech, simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
 import { ipaPhoneToKirshenbaum } from '../nlp/PhoneConversion.js'
 import { splitToWords, wordCharacterPattern } from '../nlp/Segmentation.js'
@@ -11,6 +10,7 @@ import { Lexicon, tryGetFirstLexiconSubstitution } from '../nlp/Lexicon.js'
 import { phonemizeSentence } from '../nlp/EspeakPhonemizer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { escapeHtml } from '../encodings/HtmlEscape.js'
 
 const log = logToStderr
 
@@ -433,9 +433,7 @@ export async function synthesize(text: string, espeakOptions: EspeakOptions) {
 	logger.start('Get eSpeak Emscripten instance')
 
 	if (!espeakOptions.ssml) {
-		const { escape } = await import('html-escaper')
-
-		text = escape(text)
+		text = escapeHtml(text)
 	}
 
 	const { instance } = await getEspeakInstance()
@@ -599,20 +597,6 @@ export const defaultEspeakOptions: EspeakOptions = {
 	insertSeparators: false
 }
 
-export async function testEspeakSynthesisWithPrePhonemizedInputs(text: string) {
-	const ipaPhonemizedSentence = (await phonemizeSentence(text, 'en-us')).flatMap(clause => clause)
-	const kirshenbaumPhonemizedSentence = (await phonemizeSentence(text, 'en-us', undefined, false)).flatMap(clause => clause)
-	log(kirshenbaumPhonemizedSentence)
-
-	const fragments = ipaPhonemizedSentence.map(word =>
-		word.map(phoneme =>
-			ipaPhoneToKirshenbaum(phoneme)).join('')).map(word => ` [[${word}]] `)
-
-	const { rawAudio, timeline } = await synthesizeFragments(fragments, defaultEspeakOptions)
-
-	await playAudioWithTimelinePhones(rawAudio, timeline)
-}
-
 export async function testKirshenbaumPhonemization(text: string) {
 	const ipaPhonemizedSentence = (await phonemizeSentence(text, 'en-us')).flatMap(clause => clause)
 	const kirshenbaumPhonemizedSentence = (await phonemizeSentence(text, 'en-us', undefined, false)).flatMap(clause => clause)

package/src/synthesis/FliteTTS.ts

@@ -2,10 +2,8 @@ import { SynthesisVoice } from '../api/API.js'
 import { decodeWaveToRawAudio } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { getRandomHexString, logToStderr, resolveModuleMainPath } from '../utilities/Utilities.js'
-import { readFile, open, close, remove, ensureDir } from '../utilities/FileSystem.js'
-import path from 'node:path'
-import { escape } from 'html-escaper'
-import { getAppTempDir } from '../utilities/PathUtilities.js'
+import { open, close, remove, ensureDir, readFileAsBinary, readFileAsUtf8 } from '../utilities/FileSystem.js'
+import { getAppTempDir, joinPath } from '../utilities/PathUtilities.js'
 
 const log = logToStderr
 
@@ -25,8 +23,8 @@ export async function synthesize(text: string, voice: FliteVoiceName, voiceDir:
 	const tempDir = getAppTempDir('flite')
 	await ensureDir(tempDir)
 
-	const tempAudioFilePath = path.join(tempDir, outFileName)
-	const tempStdOutFilePath = path.join(tempDir, stdOutFileName)
+	const tempAudioFilePath = joinPath(tempDir, outFileName)
+	const tempStdOutFilePath = joinPath(tempDir, stdOutFileName)
 
 	const stdOutFileFd = await open(tempStdOutFilePath, 'w+')
 
@@ -67,7 +65,7 @@ export async function synthesize(text: string, voice: FliteVoiceName, voiceDir:
 			//'-ssml',
 			'-psdur',
 			...optionArgs,
-			//` ${escape(text)} `,
+			//` ${escapeHtml(text)} `,
 			` ${text} `,
 			outFileName
 		],
@@ -100,8 +98,8 @@ export async function synthesize(text: string, voice: FliteVoiceName, voiceDir:
 		throw new Error(`Flite failed with exit code ${exitCode}`)
 	}
 
-	const waveData = await readFile(tempAudioFilePath)
-	const stdOutString = await readFile(tempStdOutFilePath, 'utf8')
+	const waveData = await readFileAsBinary(tempAudioFilePath)
+	const stdOutString = await readFileAsUtf8(tempStdOutFilePath)
 
 	await cleanup()
 
@@ -118,7 +116,7 @@ async function getModuleObject() {
 	if (!fliteModuleObject) {
 		const fliteWasiPath = await resolveModuleMainPath('@echogarden/flite-wasi')
 
-		fliteModuleObject = await WebAssembly.compile(await readFile(fliteWasiPath))
+		fliteModuleObject = await WebAssembly.compile(await readFileAsBinary(fliteWasiPath))
 	}
 
 	return fliteModuleObject

package/src/synthesis/GoogleCloudTTS.ts

@@ -1,6 +1,7 @@
 import { request } from 'gaxios'
 import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
+import { decodeBase64 } from '../encodings/Base64.js'
 
 const log = logToStderr
 
@@ -15,7 +16,7 @@ export async function synthesize(
 	ssml = false,
 	audioEncoding: AudioEncoding = 'MP3_64_KBPS',
 	sampleRate = 24000) {
-		
+
 	const logger = new Logger()
 	logger.start('Request synthesis from Google Cloud')
 
@@ -75,7 +76,7 @@ export async function synthesize(
 }
 
 function parseResponseBody(responseBody: any) {
-	const audioData = Buffer.from(responseBody.audioContent, 'base64')
+	const audioData = decodeBase64(responseBody.audioContent)
 	const timepoints: timePoint[] = responseBody.timepoints
 
 	return { audioData, timepoints }

package/src/synthesis/GoogleTranslateTTS.ts

@@ -1,12 +1,13 @@
 import { request } from 'gaxios'
 import { Phrase, splitToFragments } from '../nlp/Segmentation.js'
-import { concatBuffers, concatFloat32Arrays, logToStderr } from '../utilities/Utilities.js'
+import { concatFloat32Arrays, concatUint8Arrays, logToStderr } from '../utilities/Utilities.js'
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 import { trimAudioEnd } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { getChromeOnWindowsHeaders } from '../utilities/BrowserRequestHeaders.js'
+import { decodeBase64 } from '../encodings/Base64.js'
 
 const log = logToStderr
 
@@ -113,18 +114,18 @@ export async function synthesizeShortText(text: string, languageCode = 'en', tld
 
 	//log(response.data)
 
-	const audioChunks: Buffer[] = []
+	const audioChunks: Uint8Array[] = []
 
 	for (const line of response.data.split(/\r?\n/)) {
 		const match = line.match(/"jQ1olc","\[\\"(.*)\\"]/)
 
 		if (match != null) {
 			const base64AudioChunk = match[1]
-			audioChunks.push(Buffer.from(base64AudioChunk, 'base64'))
+			audioChunks.push(decodeBase64(base64AudioChunk))
 		}
 	}
 
-	const resultMp3Stream = concatBuffers(audioChunks)
+	const resultMp3Stream = concatUint8Arrays(audioChunks)
 
 	return resultMp3Stream
 }

package/src/synthesis/MicrosoftEdgeTTS.ts

@@ -1,17 +1,15 @@
 import { request } from 'gaxios'
 import WebSocket from 'ws'
 
-import { escape } from 'html-escaper'
-
-import splitBuffer from 'buffer-split'
-
 import * as AzureCognitiveServicesTTS from './AzureCognitiveServicesTTS.js'
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 import { Logger } from '../utilities/Logger.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
-import { concatBuffers, getRandomHexString, logToStderr } from '../utilities/Utilities.js'
+import { concatUint8Arrays, getRandomHexString, logToStderr } from '../utilities/Utilities.js'
 import { RawAudio, getEmptyRawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
 import { Timer } from '../utilities/Timer.js'
+import { decodeUtf8, encodeUtf8 } from '../encodings/Utf8.js'
+import { escapeHtml } from '../encodings/HtmlEscape.js'
 
 const traceEnabled = false
 
@@ -50,7 +48,7 @@ export async function synthesize(
 	return { rawAudio, timeline }
 }
 
-type SynthesisRequestResult = { audioData: Buffer, events: any[] }
+type SynthesisRequestResult = { audioData: Uint8Array, events: any[] }
 
 async function requestSynthesis(
 	text: string,
@@ -104,11 +102,11 @@ async function requestSynthesis(
 		throw e
 	}
 
-	const receivedBinaryMessages: Buffer[] = []
+	const receivedBinaryMessages: Uint8Array[] = []
 	const receivedEventMessages: any[] = []
-	const audioChunks: Buffer[] = []
+	const audioChunks: Uint8Array[] = []
 
-	const onMessage = (messageData: Buffer, isBinary: boolean) => {
+	const onMessage = (messageData: Uint8Array, isBinary: boolean) => {
 		lastReceievedMessageTime = Timer.currentTime
 
 		if (isBinary) {
@@ -124,11 +122,11 @@ async function requestSynthesis(
 			log(header)
 
 			receivedBinaryMessages.push(messageData)
-			receivedBinaryMessages.push(Buffer.from('\n\n'))
+			receivedBinaryMessages.push(encodeUtf8('\n\n'))
 
 			audioChunks.push(audioChunk)
 		} else {
-			const message = messageData.toString('utf8')
+			const message = decodeUtf8(messageData)
 			const [header, content] = parseTextMessage(message)
 
 			if (header['X-RequestId'] != requestId) {
@@ -147,7 +145,7 @@ async function requestSynthesis(
 				receivedEventMessages.push(content['Metadata'][0])
 			}
 			else if (header['Path'] == 'turn.end') {
-				const result: SynthesisRequestResult = { audioData: concatBuffers(audioChunks), events: receivedEventMessages }
+				const result: SynthesisRequestResult = { audioData: concatUint8Arrays(audioChunks), events: receivedEventMessages }
 
 				removeWebSocketHandlers()
 
@@ -167,7 +165,7 @@ async function requestSynthesis(
 		}
 	}
 
-	const onClose = async (code: number, reason: Buffer) => {
+	const onClose = async (code: number, reason: Uint8Array) => {
 		log('WebSocket closed.')
 		log(code)
 		log(reason.toString())
@@ -193,7 +191,7 @@ async function requestSynthesis(
 		`<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>` +
 		`<voice  name='${voice}'>` +
 		`<prosody pitch='${ssmlPitchString}' rate='${ssmlRateString}' volume='${ssmlVolumeString}'>` +
-		escape(text) +
+		escapeHtml(text) +
 		`</prosody>` +
 		`</voice>` +
 		`</speak>`
@@ -282,7 +280,7 @@ export async function initializeWebsocketConnection(trustedClientToken: string)
 		requestOpenPromise.resolve(webSocket)
 	}
 
-	const onClose = (code: number, reason: Buffer) => {
+	const onClose = (code: number, reason: Uint8Array) => {
 		log('WebSocket closed.')
 		log(code)
 		log(reason.toString())
@@ -352,11 +350,44 @@ function parseTextMessage(message: string) {
 	return [parseHeaderString(headerString), JSON.parse(content)]
 }
 
-function parseBinaryMessage(message: Buffer) {
-	const audioChunk = splitBuffer(message, Buffer.from('Path:audio\r\n'))[1]
+function parseBinaryMessage(message: Uint8Array) {
+	const patternAsBytes = encodeUtf8('Path:audio\r\n')
+	const indexOfPattern = indexOfByteSubsequence(message, patternAsBytes)
+
+	if (indexOfPattern === -1) {
+		throw new Error(`Couldn't find separator pattern in Edge TTS binary message`)
+	}
+
+	const audioChunk = message.slice(indexOfPattern + patternAsBytes.length)
+
 	const headerBuffer = message.subarray(2, message.length - audioChunk.length)
-	const headerString = headerBuffer.toString('utf8').trim()
+	const headerString = decodeUtf8(headerBuffer).trim()
 
 	return [parseHeaderString(headerString), audioChunk]
 }
 
+function indexOfByteSubsequence(sequence: Uint8Array, subsequence: Uint8Array) {
+	function tryMatchAtOffset(offset: number) {
+		if (offset + subsequence.length > sequence.length) {
+			return false
+		}
+
+		for (let i = 0; i < subsequence.length; i++) {
+			if (sequence[offset + i] !== subsequence[i]) {
+				return false
+			}
+		}
+
+		return true
+	}
+
+	for (let offset = 0; offset < sequence.length; offset++) {
+		const matched = tryMatchAtOffset(offset)
+
+		if (matched) {
+			return offset
+		}
+	}
+
+	return -1
+}

package/src/synthesis/SapiTTS.ts

@@ -107,7 +107,7 @@ export function synthesize(text: string, voiceName: string, rate = 0, useSpeechP
 			EndStream: (streamId: number, streamPos: number) => {
 				clearInterval(dispatchMessagesInterval)
 
-				const audioData = Buffer.from(sapiOutputStream.GetData())
+				const audioData = new Uint8Array(sapiOutputStream.GetData())
 				const audioChannels = decodeToChannels(audioData, 1, 16, SampleFormat.PCM)
 
 				WinAX.release(sapiOutputStream)

package/src/synthesis/StreamlabsPollyTTS.ts

@@ -92,7 +92,7 @@ export async function synthesizeFragment(text: string, voice: string) {
 	const audioUrl = responseObject.speak_url
 	const audioUrlResponse = await request<ArrayBuffer>({ url: audioUrl, responseType: 'arraybuffer' })
 
-	return Buffer.from(audioUrlResponse.data)
+	return new Uint8Array(audioUrlResponse.data)
 }
 
 export const voiceList: SynthesisVoice[] = [

package/src/synthesis/SvoxPicoTTS.ts

@@ -5,8 +5,8 @@ import { bandwidthToQFactor } from '../dsp/BiquadFilter.js'
 import { Logger } from '../utilities/Logger.js'
 import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
-import { readFile } from '../utilities/FileSystem.js'
-import { concatBuffers } from '../utilities/Utilities.js'
+import { readFileAsBinary } from '../utilities/FileSystem.js'
+import { concatUint8Arrays } from '../utilities/Utilities.js'
 
 let svoxPicoInstance: any
 
@@ -52,7 +52,7 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 	async function loadResource(localFilePath: string) {
 		const virtualFilePath = '.' + localFilePath.substring(localFilePath.lastIndexOf('/'))
 
-		const fileData = await readFile(localFilePath)
+		const fileData = await readFileAsBinary(localFilePath)
 		m.FS.writeFile(virtualFilePath, fileData)
 
 		const virtualFilePathRef = wasmMemory.allocNullTerminatedUtf8String(virtualFilePath)
@@ -109,7 +109,7 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 
 	const bytesWrittenRef = wasmMemory.allocInt32()
 
-	const audioParts: Buffer[] = []
+	const audioParts: Uint8Array[] = []
 
 	for (let textByteOffset = 0; textByteOffset < textRef.length;) {
 		bytesWrittenRef.value = 0
@@ -124,10 +124,10 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 		textByteOffset += bytesWritten
 	}
 
-	const audioData = concatBuffers(audioParts)
+	const audioData = concatUint8Arrays(audioParts)
 
 	function readAudioDataFromEngine() {
-		const outBuffers: Buffer[] = []
+		const outBuffers: Uint8Array[] = []
 
 		const outBufferLength = 16384
 		const outBufferRef = wasmMemory.allocUint8Array(outBufferLength)
@@ -148,11 +148,11 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 			}
 
 			if (outByteCount > 0) {
-				outBuffers.push(Buffer.from(outBufferRef.slice(0, outByteCount)))
+				outBuffers.push(outBufferRef.slice(0, outByteCount) as Uint8Array)
 			}
 		}
 
-		return concatBuffers(outBuffers)
+		return concatUint8Arrays(outBuffers)
 	}
 
 	dispose()

package/src/synthesis/VitsTTS.ts

@@ -5,9 +5,9 @@ import { RawAudio, getEmptyRawAudio, getRawAudioDuration } from '../audio/AudioU
 import { Lexicon } from '../nlp/Lexicon.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { readAndParseJsonFile, readdir } from '../utilities/FileSystem.js'
-import path from 'node:path'
 import { EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
 import { OnnxExecutionProvider, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js'
+import { joinPath } from '../utilities/PathUtilities.js'
 
 const cachedInstanceLookup = new Map<string, VitsTTS>()
 
@@ -196,7 +196,7 @@ export class VitsTTS {
 			throw new Error(`Couldn't file any ONNX model file in ${this.modelPath}`)
 		}
 
-		const onnxModelFilepath = path.join(this.modelPath, onnxModelFilename)
+		const onnxModelFilepath = joinPath(this.modelPath, onnxModelFilename)
 
 		const onnxSessionOptions = getOnnxSessionOptions({ executionProviders: this.executionProviders })
 

package/src/tests/Test.ts

@@ -1,6 +1,4 @@
-import { createAlignmentReferenceUsingEspeakForFragments } from '../alignment/SpeechAlignment.js'
-import { setVoice } from '../synthesis/EspeakTTS.js'
-import { concatBuffers, logToStderr, setupProgramTerminationListeners } from '../utilities/Utilities.js'
+import { logToStderr, setupProgramTerminationListeners } from '../utilities/Utilities.js'
 
 const log = logToStderr
 
@@ -9,61 +7,4 @@ setupProgramTerminationListeners()
 //process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
 //process.env.http_proxy = 'http://localhost:8080'
 
-if (false) {
-	for (let i = 0; i < 1000000; i++) {
-		if (i % 1000 === 0) {
-			console.log(`Iteration ${i}`)
-		}
-
-		await setVoice('en')
-	}
-}
-
-if (false) {
-	const fragments: string[] = []
-
-	for (let i = 0; i < 30000; i++) {
-		fragments.push(`hello`, `${i}`)
-	}
-
-	await createAlignmentReferenceUsingEspeakForFragments(fragments, {
-		voice: "gmw/en-US",
-		ssml: false,
-		rate: 150,
-		pitch: 50,
-		pitchRange: 50,
-		useKlatt: false,
-		insertSeparators: true,
-	})
-}
-
-if (false) {
-	const largeBuffer = Buffer.alloc(2 ** 33 + 1000)
-
-	largeBuffer.fill(111)
-
-	//const result = Buffer.concat([largeBuffer])
-	const result = concatBuffers([largeBuffer])
-
-	console.log(result)
-
-	console.log(result[result.length - 1])
-
-	const x = 0
-}
-
-if (false) {
-	const largeBuffer = Buffer.alloc(2 ** 33) // 8 GiB buffer
-	const { promisify } = await import('node:util')
-	const { default: gracefulFS } = await import('graceful-fs')
-	const gracefulFSWriteFile = promisify(gracefulFS.writeFile)
-	await gracefulFSWriteFile('out/test1', largeBuffer)
-}
-
-if (true) {
-	const largeBuffer = Buffer.alloc(2 ** 34) // 16 GiB buffer
-	const { writeFile } = await import('fs/promises')
-	await writeFile('out/test1', largeBuffer)
-}
-
 process.exit(0)

package/src/text-language-detection/FastTextLanguageDetection.ts

@@ -1,8 +1,8 @@
-import path from 'path'
 import { languageCodeToName } from '../utilities/Locale.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
 import { resolveModuleMainPath, roundToDigits } from '../utilities/Utilities.js'
 import { LanguageDetectionResults } from '../api/LanguageDetectionCommon.js'
+import { getDirName, joinPath } from '../utilities/PathUtilities.js'
 
 let fastTextLanguageDetectionModel: any
 
@@ -41,7 +41,7 @@ async function loadLanguageDetectionModel() {
 		const fastText = new FastText()
 
 		const moduleMainPath = await resolveModuleMainPath('@echogarden/fasttext-wasm')
-		const modelPath = path.join(path.dirname(moduleMainPath), 'lid.176.ftz')
+		const modelPath = joinPath(getDirName(moduleMainPath), 'lid.176.ftz')
 
 		const model = await fastText.loadModel(modelPath)
 

package/src/text-translation/GoogleTranslateTextTranslation.ts

@@ -197,7 +197,7 @@ export async function translateText_MobileWeb(
 		if (match) {
 			const { htmlToText } = await import('html-to-text')
 
-			translatedText = htmlToText(match[0], { wordwrap: Infinity }).trim()
+			translatedText = htmlToText(match[0], { wordwrap: false }).trim()
 		} else {
 			translatedText = ''
 		}

package/src/typings/Fillers.d.ts

@@ -21,11 +21,6 @@ declare module 'cldr-segmentation' {
 	}
 }
 
-declare module 'html-escaper' {
-	export function escape(str: string): string
-	export function unescape(str: string): string
-}
-
 declare module 'sam-js'
 declare module 'winax'
 
@@ -55,7 +50,6 @@ declare module '@echogarden/rubberband-wasm'
 declare module '@echogarden/rnnoise-wasm'
 declare module '@echogarden/fvad-wasm'
 declare module '@echogarden/sonic-wasm'
-declare module '@echogarden/kissfft-wasm'
 declare module '@echogarden/pffft-wasm'
 declare module '@echogarden/pffft-wasm/simd'
 declare module '@echogarden/speex-resampler-wasm'
@@ -92,9 +86,7 @@ declare module 'onnxruntime-node' {
 
 	type RunOptions = InferenceSession.RunOptions
 
-	/**
-	 * Binding exports a simple synchronized inference session object wrap.
-	 */
+	// Binding exports a simple synchronized inference session object wrap.
 	export namespace Binding {
 		interface InferenceSession {
 			loadModel(modelPath: string, options: SessionOptions): void
@@ -115,5 +107,5 @@ declare module 'onnxruntime-node' {
 	}
 
 	// From version.d.ts
-	export const version = '1.16.1'
+	export const version = '1.20.0'
 }

package/src/typings/TypedArray.ts

@@ -0,0 +1,5 @@
+export type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array
+
+export interface TypedArrayConstructor<T extends TypedArray> {
+	new(length: number): T
+}