echogarden 1.4.4 → 1.5.0

package/src/cli/CLI.ts

@@ -219,17 +219,21 @@ const help = [
 	`${executableName} ${chalk.magentaBright('speak-url')} url [output files...] [options...]`,
 	`    Speak the HTML document on the given URL\n`,
 	`${executableName} ${chalk.magentaBright('speak-wikipedia')} articleName [output files...] [options...]`,
-	`    Speak the given wikipedia article, language edition can be specified by --language=<langCode>\n`,
+	`    Speak the given Wikipedia article. Language edition can be specified by --language=<langCode>\n`,
 	`${executableName} ${chalk.magentaBright('transcribe')} audioFile [output files...] [options...]`,
-	`    Transcribe audio file\n`,
-	`${executableName} ${chalk.magentaBright('align')} audioFile referenceFile [output files...] [options...]`,
-	`    Align audio file to the reference transcript file\n`,
-	`${executableName} ${chalk.magentaBright('translate-speech')} inputFile [output files...] [options...]`,
+	`    Transcribe a spoken audio file\n`,
+	`${executableName} ${chalk.magentaBright('align')} audioFile transcriptFile [output files...] [options...]`,
+	`    Align spoken audio file to its transcript\n`,
+	`${executableName} ${chalk.magentaBright('translate-speech')} audioFile [output files...] [options...]`,
 	`    Transcribe audio file directly to a different language\n`,
-	`${executableName} ${chalk.magentaBright('align-translation')} audioFile referenceFile [output files...] [options...]`,
-	`    Align audio file to the reference translated transcript file\n`,
+	`${executableName} ${chalk.magentaBright('align-translation')} audioFile translatedTranscriptFile [output files...] [options...]`,
+	`    Align spoken audio file to its translated transcript\n`,
+	`${executableName} ${chalk.magentaBright('align-transcript-and-translation')} audioFile transcriptFile translatedTranscriptFile [output files...] [options...]`,
+	`    Align spoken audio file to both its transcript and its translated transcript using a two-stage approach.\n`,
+	`${executableName} ${chalk.magentaBright('align-timeline-translation')} timelineFile translatedFile [output files...] [options...]`,
+	`    Align a given timeline file to its translated text\n`,
 	`${executableName} ${chalk.magentaBright('detect-speech-language')} audioFile [output files...] [options...]`,
-	`    Detect language of audio file\n`,
+	`    Detect language of spoken audio file\n`,
 	`${executableName} ${chalk.magentaBright('detect-text-language')} inputFile [output files...] [options...]`,
 	`    Detect language of textual file\n`,
 	`${executableName} ${chalk.magentaBright('detect-voice-activity')} audioFile [output files...] [options...]`,
@@ -285,6 +289,16 @@ async function startWithArgs(operationData: CLIOperationData) {
 			break
 		}
 
+		case 'align-transcript-and-translation': {
+			await alignTranscriptAndTranslation(operationData)
+			break
+		}
+
+		case 'align-timeline-translation': {
+			await alignTimelineTranslation(operationData)
+			break
+		}
+
 		case 'detect-language': {
 			await detectLanguage(operationData, 'auto')
 			break
@@ -352,7 +366,7 @@ async function startWithArgs(operationData: CLIOperationData) {
 	}
 }
 
-async function speak(operationData: CLIOperationData) {
+export async function speak(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operation, operationOptionsLookup, cliOptions } = operationData
@@ -424,8 +438,8 @@ async function speak(operationData: CLIOperationData) {
 			textSegments = splitToParagraphs(textContent, 'single', 'preserve')
 		} else if (sourceFileExtension == 'srt' || sourceFileExtension == 'vtt') {
 			const fileContent = await readFile(sourceFile, { encoding: 'utf-8' })
-			textSegments = subtitlesToTimeline(fileContent).map(entry => entry.text)
-			//textSegments = [subtitlesToText(fileContent)]
+			//textSegments = subtitlesToTimeline(fileContent).map(entry => entry.text)
+			textSegments = [subtitlesToText(fileContent)]
 		} else if (sourceFileExtension == 'xml' || sourceFileExtension == 'ssml') {
 			options.ssml = true
 			textSegments = [fileContent]
@@ -510,7 +524,7 @@ async function speak(operationData: CLIOperationData) {
 	logger.end()
 }
 
-async function transcribe(operationData: CLIOperationData) {
+export async function transcribe(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -572,7 +586,7 @@ async function transcribe(operationData: CLIOperationData) {
 	}
 }
 
-async function align(operationData: CLIOperationData) {
+export async function align(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -668,7 +682,7 @@ async function align(operationData: CLIOperationData) {
 	}
 }
 
-async function alignTranslation(operationData: CLIOperationData) {
+export async function alignTranslation(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -677,7 +691,7 @@ async function alignTranslation(operationData: CLIOperationData) {
 	const outputFilenames = operationArgs.slice(2)
 
 	if (audioFilename == undefined) {
-		throw new Error(`align-translation requires an argument containing the audio file path.`)
+		throw new Error(`align-translation requires a first argument containing the audio file path.`)
 	}
 
 	if (!existsSync(audioFilename)) {
@@ -706,7 +720,7 @@ async function alignTranslation(operationData: CLIOperationData) {
 	} else if (referenceFileExtension == 'srt' || referenceFileExtension == 'vtt') {
 		text = subtitlesToText(fileContent)
 	} else {
-		throw new Error(`align only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
+		throw new Error(`align-translation only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
 	}
 
 	if (cliOptions.play == null) {
@@ -721,8 +735,9 @@ async function alignTranslation(operationData: CLIOperationData) {
 	const {
 		timeline,
 		wordTimeline,
-		transcript,
-		language,
+		translatedTranscript,
+		sourceLanguage,
+		targetLanguage,
 		inputRawAudio,
 		isolatedRawAudio,
 		backgroundRawAudio } = await API.alignTranslation(audioFilename, text, options)
@@ -737,10 +752,139 @@ async function alignTranslation(operationData: CLIOperationData) {
 			const segmentAudio = sliceRawAudioByTime(inputRawAudio, segmentEntry.startTime, segmentEntry.endTime)
 			const sentenceTimeline = addTimeOffsetToTimeline(segmentEntry.timeline!, -segmentEntry.startTime)
 
-			await writeOutputFilesForSegment(outputFilenames, segmentIndex, timeline.length, segmentAudio, sentenceTimeline, segmentEntry.text, language, allowOverwrite)
+			await writeOutputFilesForSegment(outputFilenames, segmentIndex, timeline.length, segmentAudio, sentenceTimeline, segmentEntry.text, targetLanguage, allowOverwrite)
+		}
+	}
+
+	for (const outputFilename of outputFilenames) {
+		const partPatternMatch = outputFilename.match(filenamePlaceholderPattern)
+
+		if (partPatternMatch) {
+			continue
+		}
+
+		const fileSaver = getFileSaver(outputFilename, allowOverwrite)
+
+		await fileSaver(inputRawAudio, timeline, translatedTranscript, options.subtitles)
+
+		await writeSourceSeparationOutputIfNeeded(outputFilename, isolatedRawAudio, backgroundRawAudio, allowOverwrite, true)
+	}
+
+	logger.end()
+
+	if (cliOptions.play) {
+		let audioToPlay: RawAudio
+
+		if (isolatedRawAudio) {
+			audioToPlay = isolatedRawAudio
+		} else {
+			audioToPlay = inputRawAudio
+		}
+
+		const normalizedAudioToPlay = normalizeAudioLevel(audioToPlay)
+
+		await playAudioWithWordTimeline(normalizedAudioToPlay, wordTimeline, translatedTranscript)
+	}
+}
+
+export async function alignTranscriptAndTranslation(operationData: CLIOperationData) {
+	const logger = new Logger()
+
+	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
+
+	const audioFilename = operationArgs[0]
+	const outputFilenames = operationArgs.slice(3)
+
+	if (audioFilename == undefined) {
+		throw new Error(`align-transcript-and-translation requires a first argument containing the audio file path.`)
+	}
+
+	if (!existsSync(audioFilename)) {
+		throw new Error(`The given source file '${audioFilename}' was not found.`)
+	}
+
+	const nativeTranscriptFilePath = operationArgs[1]
+
+	if (nativeTranscriptFilePath == undefined) {
+		throw new Error(`align-transcript-and-translation requires a second argument containing the native language transcript file path.`)
+	}
+
+	if (!existsSync(nativeTranscriptFilePath)) {
+		throw new Error(`The given transcript file '${nativeTranscriptFilePath}' was not found.`)
+	}
+
+	const translatedTranscriptFilePath = operationArgs[2]
+
+	if (translatedTranscriptFilePath == undefined) {
+		throw new Error(`align-transcript-and-translation requires a third argument containing the translated language transcript file path.`)
+	}
+
+	if (!existsSync(translatedTranscriptFilePath)) {
+		throw new Error(`The given translated transcript file '${nativeTranscriptFilePath}' was not found.`)
+	}
+
+	let transcript: string
+
+	{
+		const nativeTranscriptFileExtension = getLowercaseFileExtension(nativeTranscriptFilePath)
+		const fileContent = await readFile(nativeTranscriptFilePath, { encoding: 'utf-8' })
+
+
+		if (nativeTranscriptFileExtension == 'txt') {
+			transcript = fileContent
+		} else if (nativeTranscriptFileExtension == 'html' || nativeTranscriptFileExtension == 'htm') {
+			transcript = await convertHtmlToText(fileContent)
+		} else if (nativeTranscriptFileExtension == 'srt' || nativeTranscriptFileExtension == 'vtt') {
+			transcript = subtitlesToText(fileContent)
+		} else {
+			throw new Error(`align-transcript-and-translation only supports transcript files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
 		}
 	}
 
+	let translatedTranscript: string
+
+	{
+		const translatedTranscriptFileExtension = getLowercaseFileExtension(translatedTranscriptFilePath)
+		const fileContent = await readFile(translatedTranscriptFilePath, { encoding: 'utf-8' })
+
+		if (translatedTranscriptFileExtension == 'txt') {
+			translatedTranscript = fileContent
+		} else if (translatedTranscriptFileExtension == 'html' || translatedTranscriptFileExtension == 'htm') {
+			translatedTranscript = await convertHtmlToText(fileContent)
+		} else if (translatedTranscriptFileExtension == 'srt' || translatedTranscriptFileExtension == 'vtt') {
+			translatedTranscript = subtitlesToText(fileContent)
+		} else {
+			throw new Error(`align-transcript-and-translation only supports transcript files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
+		}
+	}
+
+	if (cliOptions.play == null) {
+		cliOptions.play = outputFilenames.length === 0
+	}
+
+	const options = await optionsLookupToTypedObject(operationOptionsLookup, 'TranscriptAndTranslationAlignmentOptions')
+
+	const allowOverwrite = getWithDefault(cliOptions.overwrite, overwriteByDefault)
+	const { includesPlaceholderPattern } = await checkOutputFilenames(outputFilenames, true, true, true)
+
+	const {
+		timeline,
+		wordTimeline,
+
+		translatedTimeline,
+		translatedWordTimeline,
+
+		sourceLanguage,
+		targetLanguage,
+
+		inputRawAudio,
+		isolatedRawAudio,
+		backgroundRawAudio } = await API.alignTranscriptAndTranslation(audioFilename, transcript, translatedTranscript, options)
+
+	if (outputFilenames.length > 0) {
+		logger.start('\nWrite output files')
+	}
+
 	for (const outputFilename of outputFilenames) {
 		const partPatternMatch = outputFilename.match(filenamePlaceholderPattern)
 
@@ -753,6 +897,16 @@ async function alignTranslation(operationData: CLIOperationData) {
 		await fileSaver(inputRawAudio, timeline, transcript, options.subtitles)
 
 		await writeSourceSeparationOutputIfNeeded(outputFilename, isolatedRawAudio, backgroundRawAudio, allowOverwrite, true)
+
+		const fileExtension = getLowercaseFileExtension(outputFilename)
+
+		if (['json', 'txt', 'srt', 'vtt'].includes(fileExtension)) {
+			const pathWithoutExtension = outputFilename.substring(0, outputFilename.lastIndexOf('.'))
+			const translatedOutputPath = `${pathWithoutExtension}.translated.${fileExtension}`
+
+			const translatedFileSaver = getFileSaver(translatedOutputPath, allowOverwrite)
+			await translatedFileSaver(inputRawAudio, translatedTimeline, translatedTranscript, options.subtitles)
+		}
 	}
 
 	logger.end()
@@ -768,11 +922,99 @@ async function alignTranslation(operationData: CLIOperationData) {
 
 		const normalizedAudioToPlay = normalizeAudioLevel(audioToPlay)
 
-		await playAudioWithWordTimeline(normalizedAudioToPlay, wordTimeline, transcript)
+		await playAudioWithWordTimeline(normalizedAudioToPlay, translatedWordTimeline, translatedTranscript)
+	}
+}
+
+export async function alignTimelineTranslation(operationData: CLIOperationData) {
+	const logger = new Logger()
+
+	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
+
+	const timelineFilename = operationArgs[0]
+	const outputFilenames = operationArgs.slice(2)
+
+	if (timelineFilename == undefined) {
+		throw new Error(`align-timeline-translation requires a first argument containing the timeline file path.`)
+	}
+
+	if (getLowercaseFileExtension(timelineFilename) != 'json') {
+		throw new Error(`align-timeline-translation only supports timeline files with extension 'json'`)
+	}
+
+	if (!existsSync(timelineFilename)) {
+		throw new Error(`The given timeline file '${timelineFilename}' was not found.`)
+	}
+
+	const timeline = await readAndParseJsonFile(timelineFilename) as Timeline
+
+	const translationFilePath = operationArgs[1]
+
+	if (translationFilePath == undefined) {
+		throw new Error(`align-timeline-translation requires a second argument containing the translated reference file path.`)
+	}
+
+	if (!existsSync(translationFilePath)) {
+		throw new Error(`The given reference file '${translationFilePath}' was not found.`)
+	}
+
+	const translationFileExtension = getLowercaseFileExtension(translationFilePath)
+	const translationFileContent = await readFile(translationFilePath, { encoding: 'utf-8' })
+
+	let translationText: string
+
+	if (translationFileExtension == 'txt') {
+		translationText = translationFileContent
+	} else if (translationFileExtension == 'html' || translationFileExtension == 'htm') {
+		translationText = await convertHtmlToText(translationFileContent)
+	} else if (translationFileExtension == 'srt' || translationFileExtension == 'vtt') {
+		translationText = subtitlesToText(translationFileContent)
+	} else {
+		throw new Error(`align only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
+	}
+
+	const options = await optionsLookupToTypedObject(operationOptionsLookup, 'TimelineTranslationAlignmentOptions')
+
+	const {
+		timeline: translationTimeline,
+		wordTimeline: translationWordTimeline,
+		rawAudio
+	} = await API.alignTimelineTranslation(timeline, translationText, options)
+
+	if (outputFilenames.length > 0) {
+		logger.start('\nWrite output files')
+	}
+
+	const allowOverwrite = getWithDefault(cliOptions.overwrite, overwriteByDefault)
+
+	for (const outputFilename of outputFilenames) {
+		const partPatternMatch = outputFilename.match(filenamePlaceholderPattern)
+
+		if (partPatternMatch) {
+			continue
+		}
+
+		const fileSaver = getFileSaver(outputFilename, allowOverwrite)
+
+		await fileSaver(getEmptyRawAudio(1, 16000), translationTimeline, translationText, options.subtitles)
+	}
+
+	logger.end()
+
+	if (cliOptions.play && rawAudio) {
+		const normalizedAudioToPlay = normalizeAudioLevel(rawAudio)
+
+		let transcriptToPlay: string
+		let timelineToPlay: Timeline
+
+		transcriptToPlay = translationText
+		timelineToPlay = translationWordTimeline
+
+		await playAudioWithWordTimeline(normalizedAudioToPlay, timelineToPlay, transcriptToPlay)
 	}
 }
 
-async function translateSpeech(operationData: CLIOperationData) {
+export async function translateSpeech(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -859,7 +1101,7 @@ async function translateSpeech(operationData: CLIOperationData) {
 	}
 }
 
-async function detectLanguage(operationData: CLIOperationData, mode: 'speech' | 'text' | 'auto') {
+export async function detectLanguage(operationData: CLIOperationData, mode: 'speech' | 'text' | 'auto') {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -936,7 +1178,7 @@ async function detectLanguage(operationData: CLIOperationData, mode: 'speech' |
 	logger.end()
 }
 
-async function detectVoiceActivity(operationData: CLIOperationData) {
+export async function detectVoiceActivity(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -1005,7 +1247,7 @@ async function detectVoiceActivity(operationData: CLIOperationData) {
 	}
 }
 
-async function denoise(operationData: CLIOperationData) {
+export async function denoise(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -1050,7 +1292,7 @@ async function denoise(operationData: CLIOperationData) {
 	}
 }
 
-async function isolate(operationData: CLIOperationData) {
+export async function isolate(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -1093,7 +1335,7 @@ async function isolate(operationData: CLIOperationData) {
 	}
 }
 
-async function listEngines(operationData: CLIOperationData) {
+export async function listEngines(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs } = operationData
@@ -1206,7 +1448,7 @@ async function listEngines(operationData: CLIOperationData) {
 	}
 }
 
-async function listTTSVoices(operationData: CLIOperationData) {
+export async function listTTSVoices(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
@@ -1268,7 +1510,7 @@ async function listTTSVoices(operationData: CLIOperationData) {
 	logger.end()
 }
 
-async function installPackages(operationData: CLIOperationData) {
+export async function installPackages(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs } = operationData
@@ -1299,7 +1541,7 @@ async function installPackages(operationData: CLIOperationData) {
 	}
 }
 
-async function uninstallPackages(operationData: CLIOperationData) {
+export async function uninstallPackages(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const { operationArgs } = operationData
@@ -1326,7 +1568,7 @@ async function uninstallPackages(operationData: CLIOperationData) {
 	}
 }
 
-async function listPackages(operationData: CLIOperationData) {
+export async function listPackages(operationData: CLIOperationData) {
 	const logger = new Logger()
 
 	const packagesDir = await ensureAndGetPackagesDir()
@@ -1358,7 +1600,7 @@ async function listPackages(operationData: CLIOperationData) {
 	logger.log(installedPackageNamesFormatted.join('\n'))
 }
 
-async function serve(operationData: CLIOperationData) {
+export async function serve(operationData: CLIOperationData) {
 	const { operationOptionsLookup } = operationData
 
 	const options = await optionsLookupToTypedObject(operationOptionsLookup, 'ServerOptions')
@@ -1500,7 +1742,7 @@ async function optionsLookupToTypedObject<K extends keyof APIOptions>(cliOptions
 }
 
 let cachedOptionsSchema: any
-export async function getOptionsSchema() {
+async function getOptionsSchema() {
 	if (!cachedOptionsSchema) {
 		cachedOptionsSchema = await readAndParseJsonFile(resolveToModuleRootDir('data/schemas/options.json'))
 	}
@@ -1508,7 +1750,7 @@ export async function getOptionsSchema() {
 	return cachedOptionsSchema
 }
 
-export async function checkOutputFilenames(outputFilenames: string[], acceptMediaOutputs: boolean, acceptMetadataOutputs: boolean, acceptSubtitleOutputs: boolean) {
+async function checkOutputFilenames(outputFilenames: string[], acceptMediaOutputs: boolean, acceptMetadataOutputs: boolean, acceptSubtitleOutputs: boolean) {
 	const supportedFileExtensions: string[] = []
 
 	if (acceptMediaOutputs) {
@@ -1562,7 +1804,7 @@ export async function checkOutputFilenames(outputFilenames: string[], acceptMedi
 	return { includesPlaceholderPattern }
 }
 
-export async function writeOutputFilesForSegment(outputFilenames: string[], index: number, total: number, audio: RawAudio, timeline: Timeline, text: string, language: string, allowOverwrite: boolean) {
+async function writeOutputFilesForSegment(outputFilenames: string[], index: number, total: number, audio: RawAudio, timeline: Timeline, text: string, language: string, allowOverwrite: boolean) {
 	const digitCount = Math.max((total + 1).toString().length, 2)
 
 	const segmentWords = (await splitToWords(text, language)).filter(text => wordCharacterPattern.test(text))

package/src/codecs/FFMpegTranscoder.ts

@@ -7,6 +7,7 @@ import { commandExists, logToStderr } from '../utilities/Utilities.js'
 import path from 'node:path'
 import { loadPackage } from '../utilities/PackageManager.js'
 import { getGlobalOption } from '../api/GlobalOptions.js'
+import { existsSync } from '../utilities/FileSystem.js'
 
 const log = logToStderr
 
@@ -67,6 +68,11 @@ async function transcode_CLI(ffmpegCommand: string, input: string | Buffer, outp
 
 		if (Buffer.isBuffer(input)) {
 			process.stdin.end(input)
+		} else if (typeof input === 'string') {
+			if (!existsSync(input)) {
+				reject(`Audio file was not found: ${input}`)
+				return
+			}
 		}
 
 		const stdoutChunks: Buffer[] = []

package/src/dsp/FFT.ts

@@ -48,7 +48,7 @@ export async function stftr(samples: Float32Array, fftOrder: number, windowSize:
 		}
 
 		binsBufferRef.clear()
-		
+
 		m._kiss_fftr(statePtr, frameBufferRef.address, binsBufferRef.address)
 
 		const bins = binsBufferRef.view.slice(0, fftOrder + 2)
@@ -229,6 +229,12 @@ export function getWindowWeights(windowType: WindowType, windowSize: number) {
 			//weights[i] = 0.54 - (0.46 * Math.cos(2 * Math.PI * (i / (windowSize - 1))))
 			weights[i] = 0.54 - (0.46 * Math.cos(innerFactor * i))
 		}
+	} else if (windowType == 'povey') {
+		const hannWeights = getWindowWeights('hann', windowSize)
+
+		for (let i = 0; i < windowSize; i++) {
+			weights[i] = hannWeights[i] ** 0.85
+		}
 	} else {
 		throw new Error(`Unsupported window function type: ${windowType}`)
 	}
@@ -255,4 +261,4 @@ export async function testFFT1(rawAudio: RawAudio) {
 	return recoveredRawAudio
 }
 
-export type WindowType = 'hann' | 'hamming'
+export type WindowType = 'hann' | 'hamming' | 'povey'

package/src/dsp/KWeightingFilter.ts

@@ -10,7 +10,7 @@ export class KWeightingFilter {
 
 		if (useStandard44100Filters) {
 			// These parameter values are taken from ITU-R BS.1770-2
-			// and designed only for a 44100 Hz sampling rate:
+			// and designed only for a 44100 Hz sample rate:
 			this.highShelfFilter = new BiquadFilter({
 				b0: 1.53512485958697,
 				b1: -2.69169618940638,

package/src/dsp/MelSpectogram.ts

@@ -2,7 +2,7 @@ import { RawAudio } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import * as FFT from './FFT.js'
 
-export async function computeMelSpectogram(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbankCount: number, lowerFrequencyHz: number, upperFrequencyHz: number) {
+export async function computeMelSpectogram(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbankCount: number, lowerFrequencyHz: number, upperFrequencyHz: number, windowType: FFT.WindowType = 'hann') {
 	const logger = new Logger()
 
 	logger.start('Compute mel filterbank')
@@ -18,15 +18,15 @@ export async function computeMelSpectogram(rawAudio: RawAudio, fftOrder: number,
 
 	logger.end()
 
-	return computeMelSpectogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, melFilterbanks)
+	return computeMelSpectogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, melFilterbanks, windowType)
 }
 
-export async function computeMelSpectogramUsingFilterbanks(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbanks: Filterbank[]) {
+export async function computeMelSpectogramUsingFilterbanks(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbanks: Filterbank[], windowType: FFT.WindowType = 'hann') {
 	const logger = new Logger()
 
 	logger.start('Compute short-time FFTs')
 	const audioSamples = rawAudio.audioChannels[0]
-	const fftFrames = await FFT.stftr(audioSamples, fftOrder, windowSize, hopLength, 'hann')
+	const fftFrames = await FFT.stftr(audioSamples, fftOrder, windowSize, hopLength, windowType)
 
 	logger.start('Convert FFT frames to a mel spectogram')
 	const melSpectogram = fftFramesToMelSpectogram(fftFrames, filterbanks)
@@ -52,17 +52,26 @@ export function powerSpectrumToMelSpectrum(powerSpectrum: Float32Array, filterba
 		const filterbankStartIndex = filterbank.startIndex
 		const filterbankWeights = filterbank.weights
 
-		if (filterbankStartIndex == -1) {
+		if (filterbankStartIndex === -1) {
 			continue
 		}
 
-		let bandValue = 0
+		let melBandValue = 0
 
 		for (let i = 0; i < filterbankWeights.length; i++) {
-			bandValue += filterbankWeights[i] * powerSpectrum[filterbankStartIndex + i]
+			const powerSpectrumIndex = filterbankStartIndex + i
+
+			if (powerSpectrumIndex >= powerSpectrum.length) {
+				break
+			}
+
+			const weight = filterbankWeights[i]
+			const powerSpectrumValue = powerSpectrum[powerSpectrumIndex]
+
+			melBandValue += weight * powerSpectrumValue
 		}
 
-		melSpectrum[melBandIndex] = bandValue
+		melSpectrum[melBandIndex] = melBandValue
 	}
 
 	return melSpectrum

package/src/math/VectorMath.ts

@@ -177,7 +177,7 @@ export function scaleToSumTo1(vector: number[]) {
 	return scaledVector
 }
 
-export function normalizeVector(vector: number[], kind: 'population' | 'sample' = 'population') {
+export function normalizeVector(vector: ArrayLike<number>, kind: 'population' | 'sample' = 'population') {
 	if (vector.length == 0) {
 		throw new Error('Vector is empty')
 	}
@@ -329,15 +329,15 @@ export function varianceOfVectors(vectors: number[][], kind: 'population' | 'sam
 	return result
 }
 
-export function meanOfVector(vector: number[]) {
+export function meanOfVector(vector: ArrayLike<number>) {
 	if (vector.length == 0) {
-		throw new Error('Vector is empty')
+		return 0
 	}
 
 	return sumVector(vector) / vector.length
 }
 
-export function medianOfVector(vector: number[]) {
+export function medianOfVector(vector: ArrayLike<number>) {
 	if (vector.length == 0) {
 		throw new Error('Vector is empty')
 	}
@@ -345,13 +345,13 @@ export function medianOfVector(vector: number[]) {
 	return vector[Math.floor(vector.length / 2)]
 }
 
-export function stdDeviationOfVector(vector: number[], kind: 'population' | 'sample' = 'population', mean?: number) {
+export function stdDeviationOfVector(vector: ArrayLike<number>, kind: 'population' | 'sample' = 'population', mean?: number) {
 	return Math.sqrt(varianceOfVector(vector, kind, mean))
 }
 
-export function varianceOfVector(vector: number[], kind: 'population' | 'sample' = 'population', mean?: number) {
+export function varianceOfVector(vector: ArrayLike<number>, kind: 'population' | 'sample' = 'population', mean?: number) {
 	if (vector.length == 0) {
-		throw new Error('Vector is empty')
+		return 0
 	}
 
 	const sampleSizeMetric = kind == 'population' || vector.length == 1 ? vector.length : vector.length - 1
@@ -362,8 +362,8 @@ export function varianceOfVector(vector: number[], kind: 'population' | 'sample'
 
 	let result = 0.0
 
-	for (const value of vector) {
-		result += (value - mean) ** 2
+	for (let i = 0; i < vector.length; i++) {
+		result += (vector[i] - mean) ** 2
 	}
 
 	return result / sampleSizeMetric