echogarden 1.0.4 → 1.1.0

package/src/server/Server.ts

@@ -7,8 +7,8 @@ import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { sendMessageToWorker, addListenerToWorkerMessages, startNewWorkerThread, startMessageChannel } from './Worker.js'
 import { Worker } from 'node:worker_threads'
 import { IncomingMessage, ServerResponse } from 'node:http'
-import { Logger } from '../utilities/Logger.js'
 import chalk from 'chalk'
+import { Logger } from '../utilities/Logger.js'
 
 const log = logToStderr
 

package/src/source-separation/MDXNetSourceSeparation.ts

@@ -4,11 +4,16 @@ import { binBufferToComplex, complexToBinBuffer, getWindowWeights, stftr, stiftr
 import { ComplexNumber } from '../math/VectorMath.js';
 import { logToStderr } from '../utilities/Utilities.js';
 import { Logger } from '../utilities/Logger.js';
+import { OnnxExecutionProvider, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js';
 
 const log = logToStderr
 
-export async function isolate(rawAudio: RawAudio, modelFilePath: string) {
-	const model = new MDXNet(modelFilePath)
+export async function isolate(
+	rawAudio: RawAudio,
+	modelFilePath: string,
+	executionProviders: OnnxExecutionProvider[]) {
+
+	const model = new MDXNet(modelFilePath, executionProviders)
 
 	return model.processAudio(rawAudio)
 }
@@ -16,7 +21,9 @@ export async function isolate(rawAudio: RawAudio, modelFilePath: string) {
 export class MDXNet {
 	session?: Onnx.InferenceSession
 
-	constructor(public readonly modelFilePath: string) {
+	constructor(
+		public readonly modelFilePath: string,
+		public readonly executionProviders: OnnxExecutionProvider[]) {
 	}
 
 	async processAudio(rawAudio: RawAudio) {
@@ -28,9 +35,7 @@ export class MDXNet {
 			throw new Error(`Input audio must have a 44100 Hz sampling rate`)
 		}
 
-		if (!this.session) {
-			await this.initializeSession(this.modelFilePath)
-		}
+		await this.initializeSessionIfNeeded()
 
 		const Onnx = await import('onnxruntime-node')
 
@@ -75,6 +80,9 @@ export class MDXNet {
 				let writePosition = 0
 
 				for (let tensorChannelIndex = 0; tensorChannelIndex < 4; tensorChannelIndex++) {
+					const isEvenTensorChannelIndex = tensorChannelIndex % 2 === 0
+					const inChannelIndex = tensorChannelIndex < 2 ? 0 : 1
+
 					for (let binIndex = 0; binIndex < fftCount; binIndex++) {
 						for (let frameIndex = 0; frameIndex < segmentSize; frameIndex++) {
 							let value = 0
@@ -82,7 +90,7 @@ export class MDXNet {
 							if (frameIndex < segmentLength && binIndex >= 0) {
 								let frame: ComplexNumber[]
 
-								if (tensorChannelIndex < 2) {
+								if (inChannelIndex === 0) {
 									frame = fftFramesLeftComplexForSegment[frameIndex]
 								} else {
 									frame = fftFramesRightComplexForSegment[frameIndex]
@@ -90,7 +98,7 @@ export class MDXNet {
 
 								const bin = frame[binIndex]
 
-								if (tensorChannelIndex % 2 === 0) {
+								if (isEvenTensorChannelIndex) {
 									value = bin.real
 								} else {
 									value = bin.imaginary
@@ -105,8 +113,12 @@ export class MDXNet {
 
 			const inputTensor = new Onnx.Tensor('float32', flattenedInputTensor, [1, 4, 2048, 256])
 
+			logger.start('Run MDXNet model')
+
 			const { output: outputTensor } = await session.run({ input: inputTensor })
 
+			logger.start('Process MDXNet model output')
+
 			const flattenedOutputTensor = outputTensor.data as Float32Array
 
 			const outputChannelComplexFrames: ComplexNumber[][][] = []
@@ -131,13 +143,16 @@ export class MDXNet {
 				let readPosition = 0
 
 				for (let tensorChannelIndex = 0; tensorChannelIndex < 4; tensorChannelIndex++) {
+					const isEvenTensorChannelIndex = tensorChannelIndex % 2 === 0
 					const outChannelIndex = tensorChannelIndex < 2 ? 0 : 1
 
+					const binsForOutputChannel = outputChannelComplexFrames[outChannelIndex]
+
 					for (let binIndex = 0; binIndex < 2048; binIndex++) {
 						for (let frameIndex = 0; frameIndex < 256; frameIndex++) {
-							const bin = outputChannelComplexFrames[outChannelIndex][frameIndex][binIndex]
+							const bin = binsForOutputChannel[frameIndex][binIndex]
 
-							if (tensorChannelIndex % 2 === 0) {
+							if (isEvenTensorChannelIndex) {
 								bin.real = flattenedOutputTensor[readPosition++]
 							} else {
 								bin.imaginary = flattenedOutputTensor[readPosition++]
@@ -149,10 +164,12 @@ export class MDXNet {
 
 			const outputAudioChannels: Float32Array[] = []
 
-			//logger.start(`Compute inverse STFT for segment`)
+			logger.start(`Compute inverse STFT for segment`)
 			for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
+				const fftSizeReciprocal = 1 / fftSize
+
 				let outputChannelFlattenedFrames = outputChannelComplexFrames[channelIndex]
-					.map(frame => complexToBinBuffer(frame).map(value => value / fftSize))
+					.map(frame => complexToBinBuffer(frame).map(value => value * fftSizeReciprocal))
 
 				const samples = await stiftr(
 					outputChannelFlattenedFrames,
@@ -164,9 +181,6 @@ export class MDXNet {
 				outputAudioChannels.push(samples)
 			}
 
-			//logger.log(`Reconstructed waveform peak: ${getAudioPeakDecibels(outputAudioChannels).toFixed(3)}dB`)
-			//await playAudioSamples({ audioChannels: outputAudioChannels, sampleRate })
-
 			audioForSegments.push(outputAudioChannels)
 		}
 
@@ -220,13 +234,15 @@ export class MDXNet {
 		return isolatedRawAudio
 	}
 
-	private async initializeSession(modelPath: string) {
-		const onnxOptions: Onnx.InferenceSession.SessionOptions = {
-			logSeverityLevel: 3
+	private async initializeSessionIfNeeded() {
+		if (this.session) {
+			return
 		}
 
 		const Onnx = await import('onnxruntime-node')
 
-		this.session = await Onnx.InferenceSession.create(modelPath, onnxOptions)
+		const onnxSessionOptions = getOnnxSessionOptions({ executionProviders: this.executionProviders })
+
+		this.session = await Onnx.InferenceSession.create(this.modelFilePath, onnxSessionOptions)
 	}
 }

package/src/speech-language-detection/SileroLanguageDetection.ts

@@ -5,10 +5,20 @@ import { RawAudio } from '../audio/AudioUtilities.js'
 import { readAndParseJsonFile } from '../utilities/FileSystem.js'
 import { detectSpeechLanguageByParts, type LanguageDetectionResults } from '../api/LanguageDetection.js'
 import { languageCodeToName } from '../utilities/Locale.js'
+import { OnnxExecutionProvider, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js'
 
-export async function detectLanguage(rawAudio: RawAudio, modelPath: string, languageDictionaryPath: string, languageGroupDictionaryPath: string) {
-	const languageDetection = new SileroLanguageDetection(modelPath, languageDictionaryPath, languageGroupDictionaryPath)
-	await languageDetection.initialize()
+export async function detectLanguage(
+	rawAudio: RawAudio,
+	modelPath: string,
+	languageDictionaryPath: string,
+	languageGroupDictionaryPath: string,
+	onnxExecutionProviders: OnnxExecutionProvider[]) {
+
+	const languageDetection = new SileroLanguageDetection(
+		modelPath,
+		languageDictionaryPath,
+		languageGroupDictionaryPath,
+		onnxExecutionProviders)
 
 	async function detectLanguageForPart(partAudio: RawAudio) {
 		const { languageResults } = await languageDetection.detectLanguage(partAudio)
@@ -24,40 +34,21 @@ export async function detectLanguage(rawAudio: RawAudio, modelPath: string, lang
 }
 
 export class SileroLanguageDetection {
-	modelPath: string
-	languageDictionaryPath: string
-	languageGroupDictionaryPath: string
-
-	languageDictionary: any
-	languageGroupDictionary: any
-
-	session: Onnx.InferenceSession | undefined
-
-	constructor(modelPath: string, languageDictionaryPath: string, languageGroupDictionaryPath: string) {
-		this.modelPath = modelPath
-		this.languageDictionaryPath = languageDictionaryPath
-		this.languageGroupDictionaryPath = languageGroupDictionaryPath
-	}
-
-	async initialize() {
-		const logger = new Logger()
-		logger.start('Initialize ONNX inference session')
-
-		this.languageDictionary = await readAndParseJsonFile(this.languageDictionaryPath)
-		this.languageGroupDictionary = await readAndParseJsonFile(this.languageGroupDictionaryPath)
-
-		const onnxOptions: Onnx.InferenceSession.SessionOptions = {
-			logSeverityLevel: 3
-		}
-
-		const Onnx = await import('onnxruntime-node')
+	languageDictionary?: any
+	languageGroupDictionary?: any
 
-		this.session = await Onnx.InferenceSession.create(this.modelPath, onnxOptions)
+	session?: Onnx.InferenceSession
 
-		logger.end()
+	constructor(
+		public readonly modelPath: string,
+		public readonly languageDictionaryPath: string,
+		public readonly languageGroupDictionaryPath: string,
+		public readonly onnxExecutionProviders: OnnxExecutionProvider[]) {
 	}
 
 	async detectLanguage(rawAudio: RawAudio) {
+		await this.initializeIfNeeded()
+
 		const logger = new Logger()
 
 		logger.start('Detect language with Silero')
@@ -65,7 +56,7 @@ export class SileroLanguageDetection {
 		const audioSamples = rawAudio.audioChannels[0]
 
 		const Onnx = await import('onnxruntime-node')
-		
+
 		const inputTensor = new Onnx.Tensor('float32', audioSamples, [1, audioSamples.length])
 
 		const inputs = { input: inputTensor }
@@ -106,4 +97,33 @@ export class SileroLanguageDetection {
 
 		return { languageResults, languageGroupResults }
 	}
+
+	async initializeIfNeeded() {
+		if (this.session) {
+			return
+		}
+
+		const logger = new Logger()
+
+		logger.start('Initialize ONNX inference session for Silero language detection')
+
+		this.languageDictionary = await readAndParseJsonFile(this.languageDictionaryPath)
+		this.languageGroupDictionary = await readAndParseJsonFile(this.languageGroupDictionaryPath)
+
+		const Onnx = await import('onnxruntime-node')
+
+		const onnxSessionOptions = getOnnxSessionOptions({ executionProviders: this.onnxExecutionProviders })
+
+		this.session = await Onnx.InferenceSession.create(this.modelPath, onnxSessionOptions)
+
+		logger.end()
+	}
+}
+
+export interface SileroLanguageDetectionOptions {
+	provider?: OnnxExecutionProvider
+}
+
+export const defaultSileroLanguageDetectionOptions: SileroLanguageDetectionOptions = {
+	provider: undefined
 }

package/src/synthesis/EspeakTTS.ts

@@ -10,6 +10,7 @@ import { splitToWords, wordCharacterPattern } from '../nlp/Segmentation.js'
 import { Lexicon, tryGetFirstLexiconSubstitution } from '../nlp/Lexicon.js'
 import { phonemizeSentence } from '../nlp/EspeakPhonemizer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
 
 const log = logToStderr
 
@@ -19,6 +20,8 @@ let espeakModule: any
 export async function preprocessAndSynthesize(text: string, language: string, espeakOptions: EspeakOptions, lexicons: Lexicon[] = []) {
 	const logger = new Logger()
 
+	espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions)
+
 	await logger.startAsync('Tokenize and analyze text')
 
 	let lowerCaseLanguageCode = language.toLowerCase()
@@ -135,6 +138,8 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 export async function synthesizeFragments(fragments: string[], espeakOptions: EspeakOptions) {
 	const logger = new Logger()
 
+	espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions)
+
 	const sampleRate = await getSampleRate()
 
 	//fragments = fragments.filter(fragment => fragment.trim() != '')
@@ -399,6 +404,9 @@ export async function synthesizeFragments(fragments: string[], espeakOptions: Es
 
 export async function synthesize(text: string, espeakOptions: EspeakOptions) {
 	const logger = new Logger()
+
+	espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions)
+
 	logger.start('Get eSpeak Emscripten instance')
 
 	if (!espeakOptions.ssml) {

package/src/synthesis/GoogleCloudTTS.ts

@@ -4,7 +4,18 @@ import { logToStderr } from '../utilities/Utilities.js'
 
 const log = logToStderr
 
-export async function synthesize(text: string, apiKey: string, languageCode = 'en-US', voice = 'en-US-Wavenet-C', speakingRate = 1.0, pitchDeltaSemitones = 0.0, volumeGainDecibels = 0.0, ssml = false, audioEncoding: AudioEncoding = 'MP3_64_KBPS', sampleRate = 24000) {
+export async function synthesize(
+	text: string,
+	apiKey: string,
+	languageCode = 'en-US',
+	voice = 'en-US-Wavenet-C',
+	speakingRate = 1.0,
+	pitchDeltaSemitones = 0.0,
+	volumeGainDecibels = 0.0,
+	ssml = false,
+	audioEncoding: AudioEncoding = 'MP3_64_KBPS',
+	sampleRate = 24000) {
+		
 	const logger = new Logger()
 	logger.start('Request synthesis from Google Cloud')
 

package/src/synthesis/VitsTTS.ts

@@ -7,17 +7,25 @@ import { Timeline } from '../utilities/Timeline.js'
 import { readAndParseJsonFile, readdir } from '../utilities/FileSystem.js'
 import path from 'node:path'
 import { EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
+import { OnnxExecutionProvider, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js'
 
 const cachedInstanceLookup = new Map<string, VitsTTS>()
 
-export async function synthesizeSentence(text: string, voiceName: string, modelPath: string, lengthScale: number, speakerId = 0, lexicons?: Lexicon[]) {
+export async function synthesizeSentence(
+	text: string,
+	voiceName: string,
+	modelPath: string,
+	lengthScale: number,
+	speakerId: number,
+	lexicons: Lexicon[],
+	executionProviders: OnnxExecutionProvider[]) {
+
 	const cacheLookupKey = modelPath
 
 	let vitsTTS: VitsTTS | undefined = cachedInstanceLookup.get(cacheLookupKey)
 
 	if (!vitsTTS) {
-		vitsTTS = new VitsTTS(voiceName, modelPath)
-		await vitsTTS.initialize()
+		vitsTTS = new VitsTTS(voiceName, modelPath, executionProviders)
 
 		cachedInstanceLookup.clear()
 		cachedInstanceLookup.set(cacheLookupKey, vitsTTS)
@@ -29,55 +37,20 @@ export async function synthesizeSentence(text: string, voiceName: string, modelP
 }
 
 export class VitsTTS {
-	voiceName: string
-	modelPath: string
-
-	modelSession?: Onnx.InferenceSession
+	session?: Onnx.InferenceSession
 	metadata?: any
 	phonemeMap?: Map<string, number[]>
 
-	constructor(voiceName: string, modelPath: string) {
-		this.voiceName = voiceName
-		this.modelPath = modelPath
-	}
-
-	async initialize() {
-		const logger = new Logger()
-		await logger.startAsync('Initialize VITS ONNX synthesis model')
-
-		const onnxOptions: Onnx.InferenceSession.SessionOptions = {
-			logSeverityLevel: 3
-		}
-
-		const { default: Onnx } = await import('onnxruntime-node')
-
-		const filesInModelPath = await readdir(this.modelPath)
-		const onnxModelFilename = filesInModelPath.find(filename => filename.endsWith('.onnx'))
-
-		if (!onnxModelFilename) {
-			throw new Error(`Couldn't file any ONNX model file in ${this.modelPath}`)
-		}
-
-		const onnxModelFilepath = path.join(this.modelPath, onnxModelFilename)
-
-		this.modelSession = await Onnx.InferenceSession.create(onnxModelFilepath, onnxOptions)
-		this.metadata = await readAndParseJsonFile(`${onnxModelFilepath}.json`)
-
-		this.phonemeMap = new Map<string, number[]>()
-
-		for (const key in this.metadata.phoneme_id_map) {
-			this.phonemeMap.set(key, this.metadata.phoneme_id_map[key])
-		}
-
-		logger.end()
+	constructor(
+		public readonly voiceName: string,
+		public readonly modelPath: string,
+		public readonly executionProviders: OnnxExecutionProvider[]) {
 	}
 
 	async synthesizeSentence(sentence: string, lengthScale: number, speakerId = 0, lexicons?: Lexicon[]) {
 		const logger = new Logger()
 
-		if (!this.modelSession) {
-			throw new Error('Model has not been initialized')
-		}
+		await this.initializeIfNeeded()
 
 		await logger.startAsync('Prepare for synthesis')
 
@@ -100,7 +73,12 @@ export class VitsTTS {
 
 		logger.end()
 
-		const espeakOptions: EspeakOptions = { ...defaultEspeakOptions, voice: espeakVoice, useKlatt: false }
+		const espeakOptions: EspeakOptions = {
+			...defaultEspeakOptions,
+			voice: espeakVoice,
+			useKlatt: false
+		}
+		
 		const { referenceSynthesizedAudio, referenceTimeline, fragments, phonemizedFragmentsSubstitutions, phonemizedSentence } = await Espeak.preprocessAndSynthesize(sentence, languageCode, espeakOptions, lexicons)
 
 		if (phonemizedSentence.length == 0) {
@@ -180,7 +158,7 @@ export class VitsTTS {
 
 		const modelInputs = { input: inputTensor, input_lengths: inputLengthsTensor, scales: scalesTensor, sid: speakerIdTensor }
 
-		const modelResults = await this.modelSession.run(modelInputs)
+		const modelResults = await this.session!.run(modelInputs)
 		const modelOutput = modelResults['output']
 
 		const modelOutputAudioSamples = modelOutput['data'] as Float32Array
@@ -200,6 +178,39 @@ export class VitsTTS {
 
 		return { rawAudio: synthesizedAudio, timeline: mappedTimeline, referenceSynthesizedAudio, referenceTimeline }
 	}
+
+	async initializeIfNeeded() {
+		if (this.session) {
+			return
+		}
+
+		const logger = new Logger()
+		await logger.startAsync('Initialize VITS ONNX synthesis model')
+
+		const { default: Onnx } = await import('onnxruntime-node')
+
+		const filesInModelPath = await readdir(this.modelPath)
+		const onnxModelFilename = filesInModelPath.find(filename => filename.endsWith('.onnx'))
+
+		if (!onnxModelFilename) {
+			throw new Error(`Couldn't file any ONNX model file in ${this.modelPath}`)
+		}
+
+		const onnxModelFilepath = path.join(this.modelPath, onnxModelFilename)
+
+		const onnxSessionOptions = getOnnxSessionOptions({ executionProviders: this.executionProviders })
+
+		this.session = await Onnx.InferenceSession.create(onnxModelFilepath, onnxSessionOptions)
+		this.metadata = await readAndParseJsonFile(`${onnxModelFilepath}.json`)
+
+		this.phonemeMap = new Map<string, number[]>()
+
+		for (const key in this.metadata.phoneme_id_map) {
+			this.phonemeMap.set(key, this.metadata.phoneme_id_map[key])
+		}
+
+		logger.end()
+	}
 }
 
 export const voiceList: SynthesisVoice[] = [

package/src/tests/Test.ts

@@ -1,4 +1,4 @@
-import { getRepetitionScoreRelativeToFirstSubstring, logToStderr, setupProgramTerminationListeners, writeToStderr } from '../utilities/Utilities.js'
+import { getTokenRepetitionScore, logToStderr, setupProgramTerminationListeners, writeToStderr } from '../utilities/Utilities.js'
 import { makeTarballsForInstalledPackages } from '../utilities/TarballMaker.js'
 import { testEspeakSynthesisWithPrePhonemizedInputs, testKirshenbaumPhonemization } from '../synthesis/EspeakTTS.js'
 

package/src/utilities/OnnxUtilities.ts

@@ -0,0 +1,68 @@
+import type * as Onnx from 'onnxruntime-node'
+
+export function getOnnxSessionOptions(options: OnnxSessionOptions) {
+
+	const onnxOptions: Onnx.InferenceSession.SessionOptions = {
+		executionProviders: ['cpu'],
+		logSeverityLevel: 3,
+	}
+
+	function dmlProviderAvailable() {
+		const platform = process.platform
+		const arch = process.arch
+
+		return platform === 'win32' && arch === 'x64'
+	}
+
+	if (options) {
+		if (options.executionProviders != null) {
+			let executionProviders = options.executionProviders.filter(provider => {
+				if (!provider) {
+					return false
+				}
+
+				if (provider === 'dml' && !dmlProviderAvailable()) {
+					return false
+				}
+
+				return true
+			})
+
+			if (!executionProviders.includes('cpu')) {
+				executionProviders.push('cpu')
+			}
+
+			executionProviders = Array.from(new Set(executionProviders))
+
+			onnxOptions.executionProviders = executionProviders as any
+		} else if (options.enableGPU === true && dmlProviderAvailable()) {
+			onnxOptions.executionProviders = ['dml', 'cpu']
+		}
+
+		if (options.logSeverityLevel != null) {
+			onnxOptions.logSeverityLevel = options.logSeverityLevel
+		}
+	}
+
+	return onnxOptions
+}
+
+export function makeOnnxLikeFloat32Tensor(onnxTensor: Onnx.Tensor): OnnxLikeFloat32Tensor {
+	return {
+		data: (onnxTensor.data as Float32Array).slice(),
+		dims: onnxTensor.dims.slice()
+	}
+}
+
+export interface OnnxLikeFloat32Tensor {
+	readonly data: Float32Array
+	readonly dims: number[]
+}
+
+export interface OnnxSessionOptions {
+	enableGPU?: boolean
+	executionProviders?: OnnxExecutionProvider[]
+	logSeverityLevel?: 0 | 1 | 2 | 3 | 4
+}
+
+export type OnnxExecutionProvider = 'cpu' | 'dml'