echogarden 1.8.6 → 2.0.0

package/src/api/Alignment.ts

@@ -92,10 +92,12 @@ export async function align(input: AudioSourceParam, transcript: string, options
 		let granularities: DtwGranularity[]
 		let windowDurations: number[]
 
-		if (typeof options.dtw!.granularity == 'string') {
-			granularities = [options.dtw!.granularity]
-		} else if (Array.isArray(options.dtw!.granularity)) {
-			granularities = options.dtw!.granularity
+		const dtwOptions = options.dtw!
+
+		if (typeof dtwOptions.granularity == 'string') {
+			granularities = [dtwOptions.granularity]
+		} else if (Array.isArray(dtwOptions.granularity)) {
+			granularities = dtwOptions.granularity
 		} else {
 			if (sourceAudioDuration < 1 * 60) {
 				// If up to 1 minute, set granularity to high, single pass
@@ -112,17 +114,68 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			}
 		}
 
-		if (options.dtw!.windowDuration) {
-			if (typeof options.dtw!.windowDuration === 'number') {
-				windowDurations = [options.dtw!.windowDuration]
-			} else if (Array.isArray(options.dtw!.windowDuration)) {
-				windowDurations = options.dtw!.windowDuration
+		if (dtwOptions.windowDuration) {
+			function tryParsePercentageWindowDuration(durationString: string) {
+				durationString = durationString.trim()
+
+				const parseResult = durationString.match(/^([0-9]+)%$/)
+
+				if (parseResult == null) {
+					throw new Error(`A DTW window duration, when provided as a string, must be formatted as an integer percentage value like '15%'.`)
+				}
+
+				const percentageValue = parseInt(parseResult[1])
+
+				if (percentageValue == null || isNaN(percentageValue) || percentageValue <= 0 || percentageValue > 100) {
+					throw new Error(`'A DTW window duration, when provided as a percentage value, must be between 0 (non-inclusive) and 100 (inclusive).`)
+				}
+
+				let durationSeconds = percentageValue / 100 * sourceAudioDuration
+				durationSeconds = Math.ceil(durationSeconds)
+				durationSeconds = Math.min(durationSeconds, sourceAudioDuration)
+
+				return durationSeconds
+			}
+
+			if (typeof dtwOptions.windowDuration === 'number') {
+				const duration = Math.min(dtwOptions.windowDuration, sourceAudioDuration)
+
+				windowDurations = [duration]
+			} else if (typeof dtwOptions.windowDuration === 'string') {
+				const durationString = dtwOptions.windowDuration.trim()
+
+				const durationSeconds = tryParsePercentageWindowDuration(durationString)
+
+				windowDurations = [durationSeconds]
+			} else if (Array.isArray(dtwOptions.windowDuration)) {
+				const durationsValues = dtwOptions.windowDuration
+
+				if (durationsValues.length < 1) {
+					throw new Error(`DTW window durations, when given as an array, must have at least one element.`)
+				}
+
+				const durations: number[] = []
+
+				for (const durationValue of durationsValues) {
+					let durationSeconds: number
+
+					if (typeof durationValue === 'number') {
+						durationSeconds = durationValue
+						durationSeconds = Math.min(durationSeconds, sourceAudioDuration)
+					} else {
+						durationSeconds = tryParsePercentageWindowDuration(durationValue)
+					}
+
+					durations.push(durationSeconds)
+				}
+
+				windowDurations = durations
 			} else {
-				throw new Error(`'dtw.windowDuration' must be a number or an array of numbers.`)
+				throw new Error(`'dtw.windowDuration' must be a number or a percentage string, or array of numbers / percentage strings.`)
 			}
 		} else {
 			if (granularities.length > 2) {
-				throw new Error(`More than two passes requested, this requires window durations to be explicitly specified for each pass. For example 'dtw.windowDuration=[600,60,10]'.`)
+				throw new Error(`More than two passes requested, this requires window durations to be explicitly specified for each pass. For example 'dtw.windowDuration=['20%',60,10]'.`)
 			}
 
 			if (sourceAudioDuration < 5 * 60) {
@@ -361,7 +414,7 @@ export interface AlignmentOptions {
 
 	dtw?: {
 		granularity?: DtwGranularity | DtwGranularity[]
-		windowDuration?: number | number[]
+		windowDuration?: number | string | (string | number)[]
 		phoneAlignmentMethod?: PhoneAlignmentMethod
 	}
 

package/src/api/Denoising.ts

@@ -1,12 +1,14 @@
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 
-import { AudioSourceParam, RawAudio, applyGainDecibelsInPlace, ensureRawAudio, getSamplePeakDecibels, mixAudio, normalizeAudioLevelInPlace } from '../audio/AudioUtilities.js'
+import { AudioSourceParam, RawAudio, applyGainDecibels, applyGainDecibelsInPlace, attenuateIfClippingInPlace, ensureRawAudio, getSamplePeakDecibels, mixAudio, normalizeAudioLevelInPlace } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 
 import { logToStderr } from '../utilities/Utilities.js'
 import { resampleAudioSpeex } from '../dsp/SpeexResampler.js'
 import { EngineMetadata } from './Common.js'
 import chalk from 'chalk'
+import { defaultNSNet2Options, NSNet2Options } from '../denoising/NSNet2.js'
+import { loadPackage } from '../utilities/PackageManager.js'
 
 const log = logToStderr
 
@@ -18,26 +20,34 @@ export async function denoise(input: AudioSourceParam, options: DenoisingOptions
 
 	const inputRawAudio = await ensureRawAudio(input)
 
-	const processingSampleRate = 48000
-
-	logger.start(`Resample audio to ${processingSampleRate} Hz`)
-	const resampledRawAudio = await resampleAudioSpeex(inputRawAudio, processingSampleRate, 3)
-
-	logger.start(`Initialize ${options.method} module`)
+	logger.start(`Initialize ${options.engine} module`)
 
 	let denoisedAudio: RawAudio
 
-	switch (options.method) {
+	switch (options.engine) {
 		case 'rnnoise': {
 			const RNNoise = await import('../denoising/RNNoise.js')
 			logger.end()
 
+			const processingSampleRate = 48000
+
+			logger.start(`Resample audio to ${processingSampleRate} Hz`)
+			const inputRawAudioResampled = await resampleAudioSpeex(inputRawAudio, processingSampleRate, 0)
+
 			const denoisedAudioChannels: Float32Array[] = []
 
-			for (const audioChannel of resampledRawAudio.audioChannels) {
+			for (let channelIndex = 0; channelIndex < inputRawAudioResampled.audioChannels.length; channelIndex++) {
+				const audioChannel = inputRawAudioResampled.audioChannels[channelIndex]
+
 				const audioChannelRawAudio: RawAudio = { audioChannels: [audioChannel], sampleRate: processingSampleRate }
 
+				logger.end()
+				logger.logTitledMessage(`Denoise audio channel`, `${channelIndex}`, chalk.magentaBright)
+
 				const { denoisedRawAudio, frameVadProbabilities } = await RNNoise.denoiseAudio(audioChannelRawAudio)
+
+				logger.end()
+
 				denoisedAudioChannels.push(denoisedRawAudio.audioChannels[0])
 			}
 
@@ -46,11 +56,68 @@ export async function denoise(input: AudioSourceParam, options: DenoisingOptions
 			break
 		}
 
+		case 'nsnet2': {
+			const NSNet2 = await import('../denoising/NSNet2.js')
+			logger.end()
+
+			const nsnet2Options = options.nsnet2!
+
+			let processingSampleRate: number
+			let packageName: string
+
+			if (nsnet2Options.model === 'baseline-16khz') {
+				processingSampleRate = 16000
+
+				packageName = 'nsnet2-20ms-baseline'
+			} else if (nsnet2Options.model === 'baseline-48khz') {
+				processingSampleRate = 48000
+
+				packageName = 'nsnet2-20ms-48k-baseline'
+			} else {
+				throw new Error(`Unknown model name: ${nsnet2Options.model}`)
+			}
+
+			if (!nsnet2Options.modelDirectoryPath) {
+				nsnet2Options.modelDirectoryPath = await loadPackage(packageName)
+			}
+
+			logger.start(`Resample audio to ${processingSampleRate} Hz`)
+			const inputRawAudioResampled = await resampleAudioSpeex(inputRawAudio, processingSampleRate, 0)
+
+			const denoisedAudioChannels: Float32Array[] = []
+
+			for (let channelIndex = 0; channelIndex < inputRawAudioResampled.audioChannels.length; channelIndex++) {
+				const audioChannel = inputRawAudioResampled.audioChannels[channelIndex]
+
+				const audioChannelRawAudio: RawAudio = { audioChannels: [audioChannel], sampleRate: processingSampleRate }
+
+				logger.end()
+
+				logger.logTitledMessage(`Denoise audio channel`, `${channelIndex}`, chalk.magentaBright)
+
+				const { denoisedAudio } = await NSNet2.denoiseAudio(audioChannelRawAudio, nsnet2Options)
+
+				logger.end()
+
+				denoisedAudioChannels.push(denoisedAudio.audioChannels[0])
+			}
+
+			denoisedAudio = { audioChannels: denoisedAudioChannels, sampleRate: processingSampleRate }
+
+			break
+		}
+
 		default: {
-			throw new Error(`Method: '${options.method}' is not supported`)
+			throw new Error(`Engine '${options.engine}' is not recognized.`)
 		}
 	}
 
+	logger.logTitledMessage(`Postprocess`, ``, chalk.magentaBright)
+
+	logger.start(`Resample denoised audio (${denoisedAudio.sampleRate} Hz) back to original sample rate (${inputRawAudio.sampleRate} Hz)`)
+
+	denoisedAudio = await ensureRawAudio(denoisedAudio, inputRawAudio.sampleRate, inputRawAudio.audioChannels.length)
+
 	logger.start('Postprocess audio')
 
 	const shouldNormalize = options.postProcessing!.normalizeAudio!
@@ -58,9 +125,11 @@ export async function denoise(input: AudioSourceParam, options: DenoisingOptions
 	const maxGainIncreaseDecibels = options.postProcessing!.maxGainIncrease!
 	const dryMixGainDecibels = options.postProcessing!.dryMixGain!
 
+	attenuateIfClippingInPlace(denoisedAudio)
+
 	const preMixPeakDecibels = getSamplePeakDecibels(denoisedAudio.audioChannels)
-	applyGainDecibelsInPlace(resampledRawAudio, dryMixGainDecibels)
-	denoisedAudio = mixAudio(denoisedAudio, resampledRawAudio)
+	const inputRawAudioWithGain = applyGainDecibels(inputRawAudio, dryMixGainDecibels)
+	denoisedAudio = mixAudio(denoisedAudio, inputRawAudioWithGain)
 	const postMixPeakDecibels = getSamplePeakDecibels(denoisedAudio.audioChannels)
 
 	if (shouldNormalize) {
@@ -85,10 +154,11 @@ export interface DenoisingResult {
 	inputRawAudio: RawAudio
 }
 
-export type DenoisingMethod = 'rnnoise'
+export type DenoisingEngine = 'rnnoise' | 'nsnet2'
 
 export interface DenoisingOptions {
-	method?: DenoisingMethod,
+	engine?: DenoisingEngine,
+
 	postProcessing?: {
 		normalizeAudio: boolean
 		targetPeak: number
@@ -96,17 +166,21 @@ export interface DenoisingOptions {
 
 		dryMixGain?: number
 	}
+
+	nsnet2?: NSNet2Options
 }
 
 export const defaultDenoisingOptions: DenoisingOptions = {
-	method: 'rnnoise',
+	engine: 'rnnoise',
 
 	postProcessing: {
 		normalizeAudio: false,
 		targetPeak: -3,
 		maxGainIncrease: 30,
-		dryMixGain: -20,
-	}
+		dryMixGain: -100,
+	},
+
+	nsnet2: defaultNSNet2Options,
 }
 
 export const denoisingEngines: EngineMetadata[] = [
@@ -115,5 +189,11 @@ export const denoisingEngines: EngineMetadata[] = [
 		name: 'RNNoise',
 		description: 'A noise suppression library based on a recurrent neural network.',
 		type: 'local'
+	},
+	{
+		id: 'nsnet2',
+		name: 'Noise Suppression Net 2',
+		description: 'Noise suppression models used as baselines for the ICASSP 2021 Deep Noise Suppression challenge.',
+		type: 'local'
 	}
 ]

package/src/api/SourceSeparation.ts

@@ -5,15 +5,13 @@ import { loadPackage } from '../utilities/PackageManager.js';
 import { EngineMetadata } from './Common.js';
 import chalk from 'chalk';
 import { readdir } from '../utilities/FileSystem.js';
-import path from 'node:path';
 import { defaultMDXNetOptions, getProfileForMDXNetModelName, MDXNetOptions } from '../source-separation/MDXNetSourceSeparation.js';
+import { joinPath } from '../utilities/PathUtilities.js';
 
 export async function isolate(input: AudioSourceParam, options: SourceSeparationOptions): Promise<SourceSeparationResult> {
 	const logger = new Logger()
 	const startTimestamp = logger.getTimestamp()
 
-	await logger.startAsync('Prepare for source separation')
-
 	const inputRawAudio = await ensureRawAudio(input)
 
 	let isolatedRawAudio: RawAudio
@@ -34,22 +32,27 @@ export async function isolate(input: AudioSourceParam, options: SourceSeparation
 				throw new Error(`Couldn't find an ONNX model file in package directory`)
 			}
 
-			const modelPath = path.join(packageDir, modelFilename)
+			const modelPath = joinPath(packageDir, modelFilename)
 
 			await logger.startAsync(`Convert audio to 44.1 kHz stereo`)
 
-			const inputRawAudio44100Stereo = await ensureRawAudio(inputRawAudio, 44100, 2)
+			let inputRawAudioAs44100Stereo: RawAudio | undefined = await ensureRawAudio(inputRawAudio, 44100, 2)
 
 			logger.end()
 
 			const modelProfile = getProfileForMDXNetModelName(mdxNetOptions.model!)
 
-			isolatedRawAudio = await MDXNetSourceSeparation.isolate(inputRawAudio44100Stereo, modelPath, modelProfile, mdxNetOptions)
-
+			isolatedRawAudio = await MDXNetSourceSeparation.isolate(inputRawAudioAs44100Stereo, modelPath, modelProfile, mdxNetOptions)
 			logger.end()
 
+			// Release memory for the converted input audio since it's not needed anymore
+			inputRawAudioAs44100Stereo = undefined
+
+			await logger.startAsync(`Convert isolated audio to back original sample rate (${inputRawAudio.sampleRate} Hz) and channel count (${inputRawAudio.audioChannels.length})`)
+			isolatedRawAudio = await ensureRawAudio(isolatedRawAudio, inputRawAudio.sampleRate, inputRawAudio.audioChannels.length)
+
 			await logger.startAsync(`Subtract from original waveform to extract background audio`)
-			backgroundRawAudio = subtractAudio(inputRawAudio44100Stereo, isolatedRawAudio)
+			backgroundRawAudio = subtractAudio(inputRawAudio, isolatedRawAudio)
 
 			break
 		}

package/src/api/SpeechLanguageDetection.ts

@@ -5,7 +5,6 @@ import { Logger } from '../utilities/Logger.js'
 
 import * as API from './API.js'
 import { logToStderr } from '../utilities/Utilities.js'
-import path from 'path'
 import { type WhisperLanguageDetectionOptions } from '../recognition/WhisperSTT.js'
 import { formatLanguageCodeWithName, languageCodeToName } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
@@ -14,6 +13,7 @@ import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
 import { type SileroLanguageDetectionOptions } from '../speech-language-detection/SileroLanguageDetection.js'
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 import { LanguageDetectionResults } from './LanguageDetectionCommon.js'
+import { joinPath } from '../utilities/PathUtilities.js'
 
 const log = logToStderr
 
@@ -55,9 +55,9 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 
 			const modelDir = await loadPackage('silero-lang-classifier-95')
 
-			const modelPath = path.join(modelDir, 'lang_classifier_95.onnx')
-			const languageDictionaryPath = path.join(modelDir, 'lang_dict_95.json')
-			const languageGroupDictionaryPath = path.join(modelDir, 'lang_group_dict_95.json')
+			const modelPath = joinPath(modelDir, 'lang_classifier_95.onnx')
+			const languageDictionaryPath = joinPath(modelDir, 'lang_dict_95.json')
+			const languageGroupDictionaryPath = joinPath(modelDir, 'lang_group_dict_95.json')
 			const onnxExecutionProviders: OnnxExecutionProvider[] = sileroOptions.provider ? [sileroOptions.provider] : []
 
 			const languageResults = await SileroLanguageDetection.detectLanguage(

package/src/api/Synthesis.ts

@@ -1,5 +1,3 @@
-import path from 'node:path'
-
 import { deepClone, extendDeep } from '../utilities/ObjectUtilities.js'
 
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
@@ -27,6 +25,8 @@ import { type ElevenlabsTTSOptions } from '../synthesis/ElevenlabsTTS.js'
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 import { simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
 import { convertHtmlToText } from '../utilities/StringUtilities.js'
+import { joinPath, resolvePath } from '../utilities/PathUtilities.js'
+import { Timer } from '../utilities/Timer.js'
 
 const log = logToStderr
 
@@ -55,6 +55,8 @@ async function synthesizeSegments(segments: string[], options: SynthesisOptions,
 	const logger = new Logger()
 	options = extendDeep(defaultSynthesisOptions, options)
 
+	const synthesisStartTime = Timer.currentTime
+
 	if (!options.language && !options.voice) {
 		logger.start('No language or voice specified. Detect language')
 
@@ -261,7 +263,7 @@ async function synthesizeSegments(segments: string[], options: SynthesisOptions,
 	async function convertToTargetCodecIfNeeded(rawAudio: RawAudio) {
 		const targetCodec = options.outputAudioFormat?.codec
 
-		let output: RawAudio | Buffer
+		let output: RawAudio | Uint8Array
 
 		if (targetCodec) {
 			logger.start(`Convert to ${targetCodec} codec`)
@@ -283,6 +285,8 @@ async function synthesizeSegments(segments: string[], options: SynthesisOptions,
 
 	logger.end()
 
+	logger.logDuration('Total synthesis time', synthesisStartTime, chalk.magentaBright)
+
 	return {
 		audio: resultAudio,
 		timeline,
@@ -292,7 +296,7 @@ async function synthesizeSegments(segments: string[], options: SynthesisOptions,
 }
 
 export interface SynthesisResult {
-	audio: RawAudio | Buffer
+	audio: RawAudio | Uint8Array
 	timeline: Timeline
 	language: string
 	voice: string
@@ -427,8 +431,8 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 			const { textAnalysisFilename, signalGenerationFilename } = SvoxPicoTTS.getResourceFilenamesForLanguage(language)
 
-			const resourceFilePath = path.resolve(voicePackagePath!, textAnalysisFilename)
-			const signalGenerationFilePath = path.resolve(voicePackagePath!, signalGenerationFilename)
+			const resourceFilePath = resolvePath(voicePackagePath!, textAnalysisFilename)
+			const signalGenerationFilePath = resolvePath(voicePackagePath!, signalGenerationFilename)
 
 			const { rawAudio } = await SvoxPicoTTS.synthesize(preparedText, resourceFilePath, signalGenerationFilePath)
 
@@ -955,7 +959,7 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 	logger.end()
 
-	logger.logDuration('Total synthesis time', startTimestamp, chalk.magentaBright)
+	logger.logDuration('Segment synthesis time', startTimestamp, chalk.magentaBright)
 
 	return { synthesizedAudio, timeline }
 }
@@ -1273,11 +1277,11 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 
 	if (!cacheDir) {
 		const appDataDir = getAppDataDir(appName)
-		cacheDir = path.join(appDataDir, 'voice-list-cache')
+		cacheDir = joinPath(appDataDir, 'voice-list-cache')
 		await ensureDir(cacheDir)
 	}
 
-	const cacheFilePath = path.join(cacheDir, `${options.engine}.voices.json`)
+	const cacheFilePath = joinPath(cacheDir, `${options.engine}.voices.json`)
 
 	async function loadVoiceList() {
 		let voiceList: SynthesisVoice[] = []
@@ -1544,7 +1548,7 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 
 
 		if (cacheFilePath) {
-			await writeFileSafe(cacheFilePath, stringifyAndFormatJson(voiceList))
+			await writeFileSafe(cacheFilePath, await stringifyAndFormatJson(voiceList))
 		}
 
 		return voiceList
@@ -1671,7 +1675,7 @@ export const defaultVoiceListRequestOptions: VoiceListRequestOptions = {
 export interface SynthesisSegmentEventData {
 	index: number
 	total: number
-	audio: RawAudio | Buffer
+	audio: RawAudio | Uint8Array
 	timeline: Timeline
 	transcript: string
 	language: string

package/src/api/VoiceActivityDetection.ts

@@ -5,13 +5,13 @@ import { AudioSourceParam, RawAudio, cropToTimeline, ensureRawAudio, } from '../
 import { Logger } from '../utilities/Logger.js'
 
 import { Timeline } from '../utilities/Timeline.js'
-import path from 'path'
 import { loadPackage } from '../utilities/PackageManager.js'
 import { EngineMetadata } from './Common.js'
 import chalk from 'chalk'
 import { type AdaptiveGateVADOptions } from '../voice-activity-detection/AdaptiveGateVAD.js'
 import { type WhisperVADOptions } from '../recognition/WhisperSTT.js'
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
+import { joinPath } from '../utilities/PathUtilities.js'
 
 const log = logToStderr
 
@@ -54,7 +54,7 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 
 			const modelDir = await loadPackage('silero-vad')
 
-			const modelPath = path.join(modelDir, 'silero-vad.onnx')
+			const modelPath = joinPath(modelDir, 'silero-vad.onnx')
 			const frameDuration = sileroOptions.frameDuration!
 
 			const onnxExecutionProviders: OnnxExecutionProvider[] = sileroOptions.provider ? [sileroOptions.provider] : []