echogarden 0.11.18 → 0.11.20

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "0.11.18",
+	"version": "0.11.20",
 	"description": "An integrated speech toolset, providing a variety of synthesis, recognition, alignment, and other processing tools, designed to be directly accessible to end-users.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -54,8 +54,8 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "^3.454.0",
-		"@aws-sdk/client-transcribe-streaming": "^3.454.0",
+		"@aws-sdk/client-polly": "^3.529.1",
+		"@aws-sdk/client-transcribe-streaming": "^3.529.1",
 		"@echogarden/espeak-ng-emscripten": "^0.1.2",
 		"@echogarden/fasttext-wasm": "^0.1.0",
 		"@echogarden/flite-wasi": "^0.1.1",
@@ -66,41 +66,42 @@
 		"@echogarden/sonic-wasm": "^0.1.1",
 		"@echogarden/speex-resampler-wasm": "^0.1.1",
 		"@echogarden/svoxpico-wasm": "^0.1.0",
-		"@mozilla/readability": "^0.4.4",
+		"@mozilla/readability": "^0.5.0",
 		"alawmulaw": "^6.0.0",
 		"buffer-split": "^1.0.0",
 		"chalk": "^5.3.0",
 		"cldr-segmentation": "^2.2.0",
 		"command-exists": "^1.2.9",
-		"compromise": "^14.10.1",
-		"compromise-dates": "^3.4.2",
-		"fs-extra": "^11.1.1",
-		"gaxios": "^6.1.1",
+		"compromise": "^14.12.0",
+		"compromise-dates": "^3.5.0",
+		"fs-extra": "^11.2.0",
+		"gaxios": "^6.3.0",
 		"graceful-fs": "^4.2.11",
 		"html-escaper": "^3.0.3",
 		"html-to-text": "^9.0.5",
 		"import-meta-resolve": "^4.0.0",
 		"jieba-wasm": "^0.0.2",
-		"jsdom": "^22.1.0",
+		"jsdom": "^24.0.0",
 		"json5": "^2.2.3",
 		"kuromoji": "^0.1.2",
-		"microsoft-cognitiveservices-speech-sdk": "^1.33.1",
+		"microsoft-cognitiveservices-speech-sdk": "^1.36.0",
 		"moving-median": "^1.0.0",
 		"msgpack-lite": "^0.1.26",
 		"ndarray": "^1.0.19",
 		"ndarray-ops": "^1.2.2",
-		"onnxruntime-node": "^1.16.3",
+		"onnxruntime-node": "^1.17.0",
 		"sam-js": "^0.2.1",
 		"strip-ansi": "^7.1.0",
 		"tar": "^6.2.0",
 		"tinyld": "^1.3.4",
-		"ws": "^8.14.2",
-		"wtf_wikipedia": "^10.2.1"
+
+		"ws": "^8.16.0",
+		"wtf_wikipedia": "^10.3.0"
 	},
 	"peerDependencies": {
 		"@echogarden/vosk": "^0.3.39-patched.1",
 		"speaker": "^0.5.4",
-		"winax": "^3.4.1"
+		"winax": "^3.4.2"
 	},
 	"peerDependenciesMeta": {
 		"@echogarden/vosk": {
@@ -121,14 +122,14 @@
 		"@types/msgpack-lite": "^0.1.11",
 		"@types/ndarray": "^1.0.14",
 		"@types/ndarray-ops": "^1.2.7",
-		"@types/node": "^20.10.0",
+		"@types/node": "^20.11.25",
 		"@types/recursive-readdir": "^2.2.4",
-		"@types/tar": "^6.1.10",
+		"@types/tar": "^6.1.11",
 		"@types/ws": "^8.5.10",
-		"@typescript-eslint/eslint-plugin": "^6.12.0",
-		"@typescript-eslint/parser": "^6.12.0",
-		"eslint": "^8.54.0",
-		"ts-json-schema-generator": "^1.4.0",
-		"typescript": "^5.3.2"
+		"@typescript-eslint/eslint-plugin": "^7.1.1",
+		"@typescript-eslint/parser": "^7.1.1",
+		"eslint": "^8.57.0",
+		"ts-json-schema-generator": "^1.5.0",
+		"typescript": "^5.4.2"
 	}
 }

package/src/api/Synthesis.ts

@@ -4,7 +4,7 @@ import { deepClone, extendDeep } from "../utilities/ObjectUtilities.js"
 
 import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
 
-import { clip, convertHtmlToText, sha256AsHex, simplifyPunctuationCharacters, stringifyAndFormatJson, logToStderr, yieldToEventLoop } from "../utilities/Utilities.js"
+import { clip, convertHtmlToText, sha256AsHex, simplifyPunctuationCharacters, stringifyAndFormatJson, logToStderr, yieldToEventLoop, delay, runOperationWithRetries } from "../utilities/Utilities.js"
 import { RawAudio, concatAudioSegments, downmixToMono, encodeWaveBuffer, getAudioPeakDecibels, getEmptyRawAudio, getRawAudioDuration, normalizeAudioLevel, trimAudioEnd, trimAudioStart } from "../audio/AudioUtilities.js"
 import { Logger } from "../utilities/Logger.js"
 
@@ -727,7 +727,10 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 			logger.end()
 
-			const { rawAudio, timeline: segmentTimeline } = await GoogleTranslateTTS.synthesizeLongText(text, language, options.googleTranslate?.tld, options.sentenceEndPause, options.segmentEndPause)
+			const { rawAudio, timeline: segmentTimeline } =
+				await runOperationWithRetries(
+					() => GoogleTranslateTTS.synthesizeLongText(text, language, options.googleTranslate?.tld, options.sentenceEndPause, options.segmentEndPause),
+					logger)
 
 			synthesizedAudio = rawAudio
 
@@ -777,7 +780,10 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 			logger.end()
 
-			const { rawAudio, timeline: edgeTimeline } = await MicrosoftEdgeTTS.synthesize(text, trustedClientToken, voice, ssmlPitch, ssmlRate)
+			const { rawAudio, timeline: edgeTimeline } =
+				await runOperationWithRetries(
+					() => MicrosoftEdgeTTS.synthesize(text, trustedClientToken!, voice, ssmlPitch, ssmlRate),
+					logger)
 
 			synthesizedAudio = rawAudio
 			timeline = edgeTimeline
@@ -1212,6 +1218,8 @@ export const defaultSynthesisOptions: SynthesisOptions = {
 export async function requestVoiceList(options: VoiceListRequestOptions): Promise<RequestVoiceListResult> {
 	options = extendDeep(defaultVoiceListRequestOptions, options)
 
+	const logger = new Logger()
+
 	const cacheOptions = options.cache!
 
 	let cacheDir = cacheOptions?.path
@@ -1456,7 +1464,10 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 					throw new Error("No trusted client token provided")
 				}
 
-				const voices = await MicrosoftEdgeTTS.getVoiceList(trustedClientToken)
+				const voices =
+					await runOperationWithRetries(
+						() => MicrosoftEdgeTTS.getVoiceList(trustedClientToken),
+						logger)
 
 				voiceList = voices.map((voice: any) => ({
 					name: voice.Name,

package/src/cli/CLI.ts

@@ -186,6 +186,8 @@ const commandHelp = [
 	`    List installed Echogarden packages\n`,
 	`${executableName} ${chalk.magentaBright('serve')} [options...]`,
 	`    Start a server\n`,
+	`Options reference: ${chalk.blueBright('https://bit.ly/echogarden-options')}`
+	//`    ${chalk.blueBright('https://github.com/echogarden-project/echogarden/blob/main/docs/Options.md')}`
 ]
 
 async function startWithArgs(parsedArgs: CLIArguments) {

package/src/math/VectorMath.ts

@@ -688,24 +688,24 @@ export function softmax(logits: number[], temperature = 1.0) {
 		}
 	}
 
-	const temperatureMultiplier = 1 / temperature
+	const temperatureReciprocal = 1 / (temperature + 1e-40)
 
 	const result: number[] = []
 
 	let sumOfExponentiatedValues = 0.0
 
 	for (const value of logits) {
-		const eToValue = Math.exp((value - maxValue) * temperatureMultiplier)
+		const eToValue = Math.exp((value - maxValue) * temperatureReciprocal)
 
 		sumOfExponentiatedValues += eToValue
 
 		result.push(eToValue)
 	}
 
-	const sumOfExponentiatedValuesMultiplier = 1 / (sumOfExponentiatedValues + 1e-40)
+	const sumOfExponentiatedValuesReciprocal = 1 / (sumOfExponentiatedValues + 1e-40)
 
 	for (let i = 0; i < result.length; i++) {
-		result[i] *= sumOfExponentiatedValuesMultiplier
+		result[i] *= sumOfExponentiatedValuesReciprocal
 	}
 
 	return result
@@ -789,6 +789,7 @@ export function sumExp(values: number[]) {
 
 export function logSoftmax(values: number[], minVal = 1e-40) {
 	const softMaxOfValues = softmax(values)
+	
 	return logOfVector(softMaxOfValues, minVal)
 }
 

package/src/server/Server.ts

@@ -4,7 +4,7 @@ import { logToStderr } from '../utilities/Utilities.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
 import { readFile, existsSync } from '../utilities/FileSystem.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
-import { sendMessageToWorker, addListenerToWorkerMessages, startNewWorkerThread } from './Worker.js'
+import { sendMessageToWorker, addListenerToWorkerMessages, startNewWorkerThread, startMessageChannel } from './Worker.js'
 import { Worker } from 'node:worker_threads'
 import { IncomingMessage, ServerResponse } from 'node:http'
 import { Logger } from '../utilities/Logger.js'
@@ -87,6 +87,7 @@ export async function startServer(serverOptions: ServerOptions, onStarted: (opti
 		workerThread = await startNewWorkerThread()
 		workerThread.on('message', onWorkerMessage)
 	} else {
+		startMessageChannel()
 		addListenerToWorkerMessages(onWorkerMessage)
 	}
 

package/src/server/Worker.ts

@@ -14,29 +14,38 @@ import chalk from "chalk"
 
 const log = logToStderr
 
-const messageChannel = new MessageChannel()
-messageChannel.port1.start()
-messageChannel.port2.start()
+let messageChannel: MessageChannel | undefined = undefined
 
 const canceledRequests = new Set<string>()
+
 let cancelCurrentTask = false
 
-export function shouldCancelCurrentTask() {
-	return cancelCurrentTask
-}
+const messageQueue = new Queue<any>()
+let isProcessing = false
 
-addListenerToClientMessages((message) => {
-	if (message.messageType == "CancelationRequest") {
-		//log(`CANCEL REQUESTED FOR ${message.requestId}`)
-		canceledRequests.add(message.requestId)
+export function startMessageChannel() {
+	if (messageChannel != null) {
 		return
 	}
 
-	enqueueAndProcessIfIdle(message)
-})
+	messageChannel = new MessageChannel()
+	messageChannel.port1.start()
+	messageChannel.port2.start()
 
-const messageQueue = new Queue<any>()
-let isProcessing = false
+	addListenerToClientMessages((message) => {
+		if (message.messageType == "CancelationRequest") {
+			//log(`CANCEL REQUESTED FOR ${message.requestId}`)
+			canceledRequests.add(message.requestId)
+			return
+		}
+
+		enqueueAndProcessIfIdle(message)
+	})
+}
+
+export function shouldCancelCurrentTask() {
+	return cancelCurrentTask
+}
 
 function enqueueAndProcessIfIdle(message: any) {
 	messageQueue.enqueue(message)
@@ -344,25 +353,39 @@ export interface TextLanguageDetectionResponseMessage extends WorkerMessageBase,
 // Messaging methods
 ///////////////////////////////////////////////////////////////////////////////////////////////
 export function sendMessageToWorker(message: any) {
-	messageChannel.port1.postMessage(message)
+	ensureMessageChannelCreated()
+
+	messageChannel?.port1.postMessage(message)
 }
 
 export function addListenerToWorkerMessages(handler: MessageFunc) {
-	messageChannel.port1.addEventListener('message', (event) => {
+	ensureMessageChannelCreated()
+
+	messageChannel?.port1.addEventListener('message', (event) => {
 		handler(event.data)
 	})
 }
 
 function sendMessageToClient(message: any) {
-	messageChannel.port2.postMessage(message)
+	ensureMessageChannelCreated()
+
+	messageChannel?.port2.postMessage(message)
 }
 
 function addListenerToClientMessages(handler: MessageFunc) {
-	messageChannel.port2.addEventListener('message', (event) => {
+	ensureMessageChannelCreated()
+
+	messageChannel?.port2.addEventListener('message', (event) => {
 		handler(event.data)
 	})
 }
 
+function ensureMessageChannelCreated() {
+	if (messageChannel == null) {
+		throw new Error(`Message channel has not been created`)
+	}
+}
+
 ///////////////////////////////////////////////////////////////////////////////////////////////
 // Worker thread methods
 ///////////////////////////////////////////////////////////////////////////////////////////////

package/src/server/WorkerStarter.ts

@@ -1,5 +1,5 @@
 import { isMainThread, parentPort } from 'node:worker_threads'
-import { sendMessageToWorker, addListenerToWorkerMessages } from './Worker.js'
+import { sendMessageToWorker, addListenerToWorkerMessages, startMessageChannel } from './Worker.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
 
 async function startIfInWorkerThread() {
@@ -7,6 +7,8 @@ async function startIfInWorkerThread() {
 		return
 	}
 
+	startMessageChannel()
+
 	addListenerToWorkerMessages((message) => {
 		parentPort?.postMessage(message)
 	})

package/src/synthesis/AzureCognitiveServicesTTS.ts

@@ -13,7 +13,7 @@ export async function synthesize(
 	subscriptionKey: string,
 	serviceRegion: string,
 	languageCode = "en-US",
-	voice = "Microsoft Server Speech Text to Speech Voice (en-US, AriaNeural)",
+	voice = "Microsoft Server Speech Text to Speech Voice (en-US, AvaNeural)",
 	ssmlEnabled = false,
 	ssmlPitchString = "+0Hz",
 	ssmlRateString = "+0%") {

package/src/synthesis/MicrosoftEdgeTTS.ts

@@ -10,7 +10,8 @@ import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
 import { Logger } from "../utilities/Logger.js"
 import { OpenPromise } from "../utilities/OpenPromise.js"
 import { getRandomHexString, logToStderr } from "../utilities/Utilities.js"
-import { getRawAudioDuration } from "../audio/AudioUtilities.js"
+import { RawAudio, getEmptyRawAudio, getRawAudioDuration } from "../audio/AudioUtilities.js"
+import { Timer } from "../utilities/Timer.js"
 
 const traceEnabled = false
 
@@ -19,7 +20,7 @@ const log: typeof logToStderr = traceEnabled ? logToStderr : () => { }
 export async function synthesize(
 	text: string,
 	trustedClientToken: string,
-	voice = "Microsoft Server Speech Text to Speech Voice (en-US, AriaNeural)",
+	voice = "Microsoft Server Speech Text to Speech Voice (en-US, AvaNeural)",
 	ssmlPitchString = "+0Hz",
 	ssmlRateString = "+0%",
 	ssmlVolumeString = "+0%") {
@@ -30,7 +31,17 @@ export async function synthesize(
 	const { audioData, events } = await requestSynthesis(text, trustedClientToken, voice, ssmlPitchString, ssmlRateString, ssmlVolumeString)
 	logger.end()
 
-	const rawAudio = await FFMpegTranscoder.decodeToChannels(audioData, 24000, 1)
+	//logToStderr(`Audio length: ${audioData.length}`)
+
+	let rawAudio: RawAudio
+
+	try {
+		rawAudio = await FFMpegTranscoder.decodeToChannels(audioData, 24000, 1)
+	} catch (e) {
+		rawAudio = getEmptyRawAudio(1, 24000)
+	}
+
+	//logToStderr(`Raw audio length: ${rawAudio.audioChannels[0].length}`)
 
 	logger.start("Convert boundary events to timeline")
 	const timeline = AzureCognitiveServicesTTS.boundaryEventsToTimeline(events, getRawAudioDuration(rawAudio))
@@ -51,15 +62,55 @@ async function requestSynthesis(
 
 	const synthesisOpenPromise = new OpenPromise<SynthesisRequestResult>()
 
+	let connectionFailed = false
+	let responseComplete = false
+
+	let lastReceievedMessageTime = Timer.currentTime
+
+	function checkForRequestTimeout() {
+		if (responseComplete || connectionFailed) {
+			clearInterval(timeoutCheckInterval)
+
+			return
+		}
+
+		if (!webSocket) {
+			return
+		}
+
+		if (Timer.currentTime - lastReceievedMessageTime > 10000) {
+			clearInterval(timeoutCheckInterval)
+
+			removeWebSocketHandlers()
+			webSocket.close()
+
+			synthesisOpenPromise.reject(new Error(`WebSocket request timed out after 10 seconds without a message from the server`))
+
+			return
+		}
+	}
+
+	let webSocket: WebSocket = undefined as any
+
+	const timeoutCheckInterval = setInterval(checkForRequestTimeout, 1000)
+
 	const requestId = getRandomHexString()
 
-	const webSocket = await initializeWebsocketConnection(trustedClientToken)
+	try {
+		webSocket = await initializeWebsocketConnection(trustedClientToken)
+	} catch(e) {
+		connectionFailed = true
+
+		throw e
+	}
 
 	const receivedBinaryMessages: Buffer[] = []
 	const receivedEventMessages: any[] = []
 	const audioChunks: Buffer[] = []
 
 	const onMessage = (messageData: Buffer, isBinary: boolean) => {
+		lastReceievedMessageTime = Timer.currentTime
+
 		if (isBinary) {
 			const [header, audioChunk] = parseBinaryMessage(messageData)
 
@@ -98,17 +149,22 @@ async function requestSynthesis(
 			else if (header["Path"] == "turn.end") {
 				const result: SynthesisRequestResult = { audioData: Buffer.concat(audioChunks), events: receivedEventMessages }
 
-				webSocket.off("message", onMessage)
-				webSocket.off("error", onError)
-				webSocket.off("close", onClose)
+				removeWebSocketHandlers()
 
 				synthesisOpenPromise.resolve(result)
+				responseComplete = true
 			}
 		}
 	}
 
 	const onError = (err: Error) => {
 		synthesisOpenPromise.reject(err)
+
+		removeWebSocketHandlers()
+
+		if (webSocket) {
+			webSocket.close()
+		}
 	}
 
 	const onClose = async (code: number, reason: Buffer) => {
@@ -117,7 +173,15 @@ async function requestSynthesis(
 		log(reason.toString())
 
 		if (reason.length > 0) {
-			synthesisOpenPromise.reject(`Websocket closed with code ${code}, reason: ${reason}`)
+			synthesisOpenPromise.reject(new Error(`Websocket closed with code ${code}, reason: ${reason}`))
+		}
+	}
+
+	function removeWebSocketHandlers() {
+		if (webSocket) {
+			webSocket.off("message", onMessage)
+			webSocket.off("error", onError)
+			webSocket.off("close", onClose)
 		}
 	}
 
@@ -224,7 +288,7 @@ export async function initializeWebsocketConnection(trustedClientToken: string)
 		log(reason.toString())
 
 		if (reason.length > 0) {
-			requestOpenPromise.reject(`Websocket closed with code ${code}, reason: ${reason}`)
+			requestOpenPromise.reject(new Error(`Websocket closed with code ${code}, reason: ${reason}`))
 		}
 	}
 

package/src/utilities/Logger.ts

@@ -24,9 +24,20 @@ export class Logger {
 		}
 
 		writeToStderr(`${titleColor(title)}.. `)
+
+		this.setAsActiveLogger()
+		
+		this.timer.restart()
+	}
+
+	setAsActiveLogger() {
 		this.active = true
 		currentActiveLogger = this
-		this.timer.restart()
+	}
+
+	unsetAsActiveLogger() {
+		this.active = false
+		currentActiveLogger = null
 	}
 
 	end() {

package/src/utilities/Utilities.ts

@@ -4,6 +4,8 @@ import { inspect } from 'node:util'
 
 import { RandomGenerator } from './RandomGenerator.js'
 import { randomUUID, randomBytes } from 'node:crypto'
+import { Logger } from './Logger.js'
+import chalk from 'chalk'
 
 const log = logToStderr
 
@@ -445,9 +447,9 @@ export function splitFilenameOnExtendedExtension(filenameWithExtension: string)
 				splitPoint = i
 
 				continue
-			 } else {
-			 	break
-			 }
+			} else {
+				break
+			}
 		}
 	}
 
@@ -566,9 +568,40 @@ export function getConsecutiveRepetitionScoreRelativeToFirstSubstring(tokens: st
 export async function resolveModuleScriptPath(moduleName: string) {
 	const { resolve } = await import('import-meta-resolve')
 
-	const scriptPath = await resolve(moduleName, import.meta.url)
+	const scriptPath = resolve(moduleName, import.meta.url)
 
 	const { fileURLToPath } = await import('url')
 
 	return fileURLToPath(scriptPath)
 }
+
+export async function runOperationWithRetries<R>(
+	operationFunc: () => Promise<R>,
+	logger: Logger,
+	opreationName = "Operation",
+	delayBetweenRetries = 2000,
+	maxRetries = 1000) {
+
+	for (let retryIndex = 1; retryIndex <= maxRetries; retryIndex++) {
+		try {
+			const result = await operationFunc()
+
+			return result
+		} catch (e: any) {
+			logger.setAsActiveLogger()
+
+			logger.logTitledMessage(`Error`, e.message, chalk.redBright)
+			logger.log(``)
+			logger.logTitledMessage(`${opreationName} failed`, `Trying again in ${delayBetweenRetries}ms..`, chalk.redBright)
+
+			await delay(delayBetweenRetries)
+
+			logger.log(``)
+			logger.logTitledMessage(`Starting attempt`, `${retryIndex} / ${maxRetries}`)
+
+			logger.unsetAsActiveLogger()
+		}
+	}
+
+	throw new Error(`${opreationName} failed after ${maxRetries} retry attempts`)
+}