echogarden 1.6.2 → 1.7.0

package/src/api/SourceSeparation.ts

@@ -7,12 +7,13 @@ import chalk from 'chalk';
 import { readdir } from '../utilities/FileSystem.js';
 import path from 'node:path';
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js';
+import { getProfileForMDXNetModelName, MDXNetModelName } from '../source-separation/MDXNetSourceSeparation.js';
 
 export async function isolate(input: AudioSourceParam, options: SourceSeparationOptions): Promise<SourceSeparationResult> {
 	const logger = new Logger()
 	const startTimestamp = logger.getTimestamp()
 
-	logger.start('Prepare for source separation')
+	await logger.startAsync('Prepare for source separation')
 
 	const inputRawAudio = await ensureRawAudio(input)
 
@@ -27,7 +28,8 @@ export async function isolate(input: AudioSourceParam, options: SourceSeparation
 
 			const mdxNetOptions = options.mdxNet!
 
-			const executionProviders: OnnxExecutionProvider[] = mdxNetOptions.executionProvider ? [mdxNetOptions.executionProvider] : []
+			const executionProviders: OnnxExecutionProvider[] =
+				mdxNetOptions.provider ? [mdxNetOptions.provider] : MDXNetSourceSeparation.getDefaultMDXNetProviders()
 
 			const packageDir = await loadPackage(`mdxnet-${mdxNetOptions.model!}`)
 			const modelFilename = (await readdir(packageDir)).filter(name => name.endsWith('onnx'))[0]
@@ -38,16 +40,20 @@ export async function isolate(input: AudioSourceParam, options: SourceSeparation
 
 			const modelPath = path.join(packageDir, modelFilename)
 
+			await logger.startAsync(`Convert audio to 44.1 kHz stereo`)
+
+			const inputRawAudio44100Stereo = await ensureRawAudio(inputRawAudio, 44100, 2)
+
 			logger.end()
 
-			const audioStereo44100 = await ensureRawAudio(inputRawAudio, 44100, 2)
+			const modelProfile = getProfileForMDXNetModelName(mdxNetOptions.model!)
 
-			isolatedRawAudio = await MDXNetSourceSeparation.isolate(audioStereo44100, modelPath, executionProviders)
+			isolatedRawAudio = await MDXNetSourceSeparation.isolate(inputRawAudio44100Stereo, modelPath, modelProfile, executionProviders)
 
 			logger.end()
 
-			logger.start(`Subtract from original waveform to extract background audio`)
-			backgroundRawAudio = subtractAudio(audioStereo44100, isolatedRawAudio)
+			await logger.startAsync(`Subtract from original waveform to extract background audio`)
+			backgroundRawAudio = subtractAudio(inputRawAudio44100Stereo, isolatedRawAudio)
 
 			break
 		}
@@ -58,7 +64,6 @@ export async function isolate(input: AudioSourceParam, options: SourceSeparation
 	}
 
 	logger.end()
-	logger.log('')
 	logger.logDuration(`Total source separation time`, startTimestamp, chalk.magentaBright)
 
 	return {
@@ -74,8 +79,8 @@ export interface SourceSeparationOptions {
 	engine?: SourceSeparationEngine
 
 	mdxNet?: {
-		model?: string
-		executionProvider?: OnnxExecutionProvider
+		model?: MDXNetModelName
+		provider?: OnnxExecutionProvider
 	}
 }
 
@@ -84,7 +89,7 @@ export const defaultSourceSeparationOptions: SourceSeparationOptions = {
 
 	mdxNet: {
 		model: 'UVR_MDXNET_1_9703',
-		executionProvider: undefined,
+		provider: undefined,
 	}
 }
 

package/src/api/Synthesis.ts

@@ -4,7 +4,7 @@ import { deepClone, extendDeep } from '../utilities/ObjectUtilities.js'
 
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 
-import { clip, convertHtmlToText, sha256AsHex, simplifyPunctuationCharacters, stringifyAndFormatJson, logToStderr, yieldToEventLoop, runOperationWithRetries } from '../utilities/Utilities.js'
+import { clip, sha256AsHex, stringifyAndFormatJson, logToStderr, yieldToEventLoop, runOperationWithRetries } from '../utilities/Utilities.js'
 import { RawAudio, attenuateIfClipping, concatAudioSegments, downmixToMono, encodeRawAudioToWave, getSamplePeakDecibels, getEmptyRawAudio, getRawAudioDuration, normalizeAudioLevel, trimAudioEnd, trimAudioStart } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 
@@ -25,6 +25,8 @@ import { type EspeakOptions } from '../synthesis/EspeakTTS.js'
 import { type OpenAICloudTTSOptions  } from '../synthesis/OpenAICloudTTS.js'
 import { type ElevenlabsTTSOptions } from '../synthesis/ElevenlabsTTS.js'
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
+import { simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
+import { convertHtmlToText } from '../utilities/StringUtilities.js'
 
 const log = logToStderr
 

package/src/api/TextTranslation.ts

@@ -3,6 +3,7 @@ import { formatLanguageCodeWithName, normalizeIdentifierToLanguageCode, parseLan
 import { Logger } from '../utilities/Logger.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import * as API from './API.js'
+import { defaultGoogleTranslateTextTranslationOptions, type GoogleTranslateTextTranslationOptions } from '../text-translation/GoogleTranslateTextTranslation.js'
 
 export async function translateText(inputText: string, options: TextTranslationOptions): Promise<TextTranslationResult> {
 	const logger = new Logger()
@@ -43,7 +44,7 @@ export async function translateText(inputText: string, options: TextTranslationO
 
 			logger.end()
 
-			logger.logTitledMessage(`Warning`, `The nllb engine is currently an early prototype implementation and doesn't work correctly.`, chalk.yellow, 'warning')
+			logger.logTitledMessage(`Warning`, `The nllb text translation engine is currently a work-in-progress and doesn't work correctly.`, chalk.yellow, 'warning')
 
 			translationPairs = await NLLBTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage)
 
@@ -63,9 +64,11 @@ export async function translateText(inputText: string, options: TextTranslationO
 		case 'google-translate': {
 			const GoogleTranslateTextTranslation = await import('../text-translation/GoogleTranslateTextTranslation.js')
 
+			const googleTranslateOptions = options.googleTranslate!
+
 			logger.end();
 
-			({ translationPairs, translatedText } = await GoogleTranslateTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage))
+			({ translationPairs, translatedText } = await GoogleTranslateTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage, options.plainText!, googleTranslateOptions))
 
 			break
 		}
@@ -75,7 +78,7 @@ export async function translateText(inputText: string, options: TextTranslationO
 
 			logger.end()
 
-			logger.logTitledMessage(`Warning`, `The deepl engine is currently an early prototype implementation and doesn't work correctly.`, chalk.yellow, 'warning')
+			logger.logTitledMessage(`Warning`, `The deepl text translation engine is currently a work-in-progress and doesn't work correctly.`, chalk.yellow, 'warning')
 
 			translationPairs = await DeepLTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage)
 			translatedText = ''
@@ -113,11 +116,12 @@ export interface TextTranslationOptions {
 
 	languageDetection?: API.TextLanguageDetectionOptions
 
+	plainText?: API.PlainTextOptions
+
 	nllb?: {
 	},
 
-	googleTranslate?: {
-	},
+	googleTranslate?: GoogleTranslateTextTranslationOptions,
 
 	deepl?: {
 	},
@@ -148,12 +152,16 @@ export const defaultTextTranslationOptions: TextTranslationOptions = {
 
 	languageDetection: undefined,
 
-	nllb: {
+	plainText: {
+		paragraphBreaks: 'double',
+		whitespace: 'preserve'
 	},
 
-	googleTranslate: {
+	nllb: {
 	},
 
+	googleTranslate: defaultGoogleTranslateTextTranslationOptions,
+
 	deepl: {
 	},
 }
@@ -162,7 +170,7 @@ export const textTranslationEngines: API.EngineMetadata[] = [
 	{
 		id: 'nllb',
 		name: 'NLLB',
-		description: 'No Language Left Behind (NLLB) is a deep learning machine translation model by Facebook Research (early prototype implementation).',
+		description: 'No Language Left Behind (NLLB) is a deep learning machine translation model by Facebook Research (work-in-progress, do not use).',
 		type: 'local'
 	},
 	{
@@ -174,7 +182,7 @@ export const textTranslationEngines: API.EngineMetadata[] = [
 	{
 		id: 'deepl',
 		name: 'DeepL',
-		description: 'Unoffical text translation API used by the DeepL web interface (early prototype implementation).',
+		description: 'Unoffical text translation API used by the DeepL web interface (work-in-progress, do not use).',
 		type: 'cloud'
 	},
 ]

package/src/build-tools/MakeTarballsForInstalledPackages.ts

@@ -0,0 +1,25 @@
+import path from 'path'
+import { existsSync, readdir } from '../utilities/FileSystem.js'
+import { ensureAndGetPackagesDir } from '../utilities/PackageManager.js'
+import { createNamedTarball } from '../utilities/TarballMaker.js'
+import { setupProgramTerminationListeners } from '../utilities/Utilities.js'
+
+async function makeTarballsForInstalledPackages(tarballDir: string, skipIfExists = false) {
+	const packagesDir = await ensureAndGetPackagesDir()
+	const packageList = await readdir(packagesDir)
+
+	for (const packageName of packageList) {
+		if (skipIfExists && existsSync(path.join(tarballDir, `${packageName}.tar.gz`))) {
+			continue
+		}
+
+		const packagePath = path.join(packagesDir, packageName)
+		await createNamedTarball(packagePath, packageName, tarballDir)
+	}
+}
+
+setupProgramTerminationListeners()
+
+await makeTarballsForInstalledPackages('../resources/tarballs', true)
+
+process.exit(0)

package/src/cli/CLI.ts

@@ -1,6 +1,6 @@
 import * as API from '../api/API.js'
 import { parseCLIArguments } from './CLIParser.js'
-import { convertHtmlToText, formatIntegerWithLeadingZeros, formatListWithQuotedElements, getWithDefault, logToStderr, setupUnhandledExceptionListeners, splitFilenameOnExtendedExtension, stringifyAndFormatJson } from '../utilities/Utilities.js'
+import { getWithDefault, logToStderr, setupUnhandledExceptionListeners, splitFilenameOnExtendedExtension, stringifyAndFormatJson } from '../utilities/Utilities.js'
 import { getOptionTypeFromSchema, SchemaTypeDefinition } from './CLIOptionsSchema.js'
 import { ParsedConfigFile, parseConfigFile, parseJSONConfigFile } from './CLIConfigFile.js'
 
@@ -26,6 +26,7 @@ import { OpenPromise } from '../utilities/OpenPromise.js'
 import JSON5 from 'json5'
 import { getLowercaseFileExtension, resolveToModuleRootDir } from '../utilities/PathUtilities.js'
 import { CLIOptions, CLIOptionsKeys } from './CLIOptions.js'
+import { convertHtmlToText, formatIntegerWithLeadingZeros, formatListWithQuotedElements } from '../utilities/StringUtilities.js'
 
 //const log = logToStderr
 
@@ -1049,7 +1050,7 @@ export async function translateText(operationData: CLIOperationData) {
 	} else if (inputFileExtension == 'srt' || inputFileExtension == 'vtt') {
 		inputText = subtitlesToText(inputFileContent)
 	} else {
-		throw new Error(`align only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
+		throw new Error(`translate-text only supports input files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
 	}
 
 	const options = await optionsLookupToTypedObject(operationOptionsLookup, 'TextTranslationOptions')
@@ -1066,7 +1067,7 @@ export async function translateText(operationData: CLIOperationData) {
 
 		sourceLanguage,
 		targetLanguage,
-	} = await API.translateText(inputFileContent, options)
+	} = await API.translateText(inputText, options)
 
 	if (outputFilenames.length > 0) {
 		logger.start('\nWrite output files')

package/src/dsp/FFT.ts

@@ -1,13 +1,24 @@
-import { RawAudio } from '../audio/AudioUtilities.js'
-import { ComplexNumber, sumVector } from '../math/VectorMath.js'
+import { ComplexNumber } from '../math/VectorMath.js'
 import { concatFloat32Arrays } from '../utilities/Utilities.js'
 import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js'
 
 let kissFFTInstance: any
 
+// Compute short-term Fourier transform (real-valued)
 export async function stftr(samples: Float32Array, fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType) {
+	const frames: Float32Array[] = []
+
+	for await (const frame of stftrGenerator(samples, fftOrder, windowSize, hopSize, windowType)) {
+		frames.push(frame)
+	}
+
+	return frames
+}
+
+// Incrementally generate short-term Fourier transform frames (real-valued)
+export async function* stftrGenerator(samples: Float32Array, fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType) {
 	if (fftOrder % 2 != 0 || windowSize % 2 != 0) {
-		throw new Error('FFT order and window size must multiples of 2')
+		throw new Error('FFT order and window size must be multiples of 2')
 	}
 
 	if (windowSize > fftOrder) {
@@ -35,8 +46,6 @@ export async function stftr(samples: Float32Array, fftOrder: number, windowSize:
 	const frameBufferRef = wasmMemory.allocFloat32Array(fftOrder)
 	const binsBufferRef = wasmMemory.allocFloat32Array(fftOrder * 2)
 
-	const frames: Float32Array[] = []
-
 	for (let offset = 0; offset < sampleCount; offset += hopSize) {
 		const windowSamples = samples.subarray(offset, offset + windowSize)
 		frameBufferRef.clear()
@@ -52,14 +61,14 @@ export async function stftr(samples: Float32Array, fftOrder: number, windowSize:
 		m._kiss_fftr(statePtr, frameBufferRef.address, binsBufferRef.address)
 
 		const bins = binsBufferRef.view.slice(0, fftOrder + 2)
-		frames.push(bins)
+
+		yield bins
 	}
 
 	wasmMemory.freeAll()
-
-	return frames
 }
 
+// Compute short-term inverse Fourier transform (real-valued)
 export async function stiftr(binsForFrames: Float32Array[], fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType, expectedOutputLength?: number) {
 	if (fftOrder % 2 != 0 || windowSize % 2 != 0) {
 		throw new Error('FFT order and window size must multiples of 2')
@@ -135,6 +144,7 @@ export async function stiftr(binsForFrames: Float32Array[], fftOrder: number, wi
 	return outSamplesTrimmed
 }
 
+// Get bin frequency thresholds for a particular bin count and maximum frequency
 export function getBinFrequencies(binCount: number, maxFrequency: number) {
 	const binFrequencies = new Float32Array(binCount)
 	const frequencyStep = maxFrequency / (binCount - 1)
@@ -146,10 +156,12 @@ export function getBinFrequencies(binCount: number, maxFrequency: number) {
 	return binFrequencies
 }
 
+// Convert an array of raw FFT frames to a power spectrum
 export function fftFramesToPowerSpectogram(fftFrames: Float32Array[]) {
 	return fftFrames.map(fftFrame => fftFrameToPowerSpectrum(fftFrame))
 }
 
+// Convert a raw FFT frame to a power spectrum
 export function fftFrameToPowerSpectrum(fftFrame: Float32Array) {
 	const powerSpectrum = new Float32Array(fftFrame.length / 2)
 
@@ -165,6 +177,7 @@ export function fftFrameToPowerSpectrum(fftFrame: Float32Array) {
 	return powerSpectrum
 }
 
+// Convert raw FFT frames to an array of complex numbers
 export function binBufferToComplex(bins: Float32Array, extendAndMirror = false) {
 	const complexBins: ComplexNumber[] = []
 
@@ -191,6 +204,7 @@ export function binBufferToComplex(bins: Float32Array, extendAndMirror = false)
 	return complexBins
 }
 
+// Convert an array of complex numbers to raw FFT frames
 export function complexToBinBuffer(complexBins: ComplexNumber[]) {
 	const binBuffer = new Float32Array(complexBins.length * 2)
 
@@ -204,16 +218,7 @@ export function complexToBinBuffer(complexBins: ComplexNumber[]) {
 	return binBuffer
 }
 
-export async function getKissFFTInstance() {
-	if (!kissFFTInstance) {
-		const { default: initializer } = await import('@echogarden/kissfft-wasm')
-
-		kissFFTInstance = await initializer()
-	}
-
-	return kissFFTInstance
-}
-
+// Get window weights for a particular window function
 export function getWindowWeights(windowType: WindowType, windowSize: number) {
 	const weights = new Float32Array(windowSize)
 
@@ -242,23 +247,15 @@ export function getWindowWeights(windowType: WindowType, windowSize: number) {
 	return weights
 }
 
-export async function testFFT1(rawAudio: RawAudio) {
-	const { resampleAudioSpeex } = await import('./SpeexResampler.js')
-
-	const samples = (await resampleAudioSpeex(rawAudio, 16000)).audioChannels[0]
-
-	const fftOrder = 512
-	const windowSize = 320
-	const hopLength = windowSize / 2
-	const windowType: WindowType = 'hann'
-
-	const bins = await stftr(samples, fftOrder, windowSize, hopLength, windowType)
-	const normalizedBins = bins.map(bin => bin.map(x => x / fftOrder))
-	const recoveredSamples = await stiftr(normalizedBins, fftOrder, windowSize, hopLength, windowType, samples.length)
+// Get KISS FFT instance (initialize new if not exists)
+export async function getKissFFTInstance() {
+	if (!kissFFTInstance) {
+		const { default: initializer } = await import('@echogarden/kissfft-wasm')
 
-	const recoveredRawAudio: RawAudio = { audioChannels: [recoveredSamples], sampleRate: 16000 }
+		kissFFTInstance = await initializer()
+	}
 
-	return recoveredRawAudio
+	return kissFFTInstance
 }
 
 export type WindowType = 'hann' | 'hamming' | 'povey'

package/src/nlp/Segmentation.ts

@@ -1,10 +1,11 @@
 import * as CldrSegmentation from 'cldr-segmentation'
 import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js'
 
-import { sumArray, includesAnyOf, indexOfAnyOf, logToStderr } from '../utilities/Utilities.js'
+import { sumArray, logToStderr } from '../utilities/Utilities.js'
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js'
 import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
+import { includesAnyOf, indexOfAnyOf } from '../utilities/StringUtilities.js'
 
 const log = logToStderr
 

package/src/nlp/TextNormalizer.ts

@@ -1,4 +1,5 @@
 import { getShortLanguageCode } from '../utilities/Locale.js'
+import { substituteCharactersUsingLookup } from '../utilities/StringUtilities.js'
 
 export function getNormalizedFragmentsForSpeech(words: string[], language: string) {
 	language = getShortLanguageCode(language)
@@ -158,3 +159,62 @@ export function normalizeFourDigitDecadeString(decadeString: string) {
 
 	return normalizedString
 }
+
+export function simplifyPunctuationCharacters(text: string) {
+	return substituteCharactersUsingLookup(text, punctuationSubstitutionLookup)
+}
+
+export const punctuationSubstitutionLookup: Record<string, string> = {
+	'“': `"`,
+	'”': `"`,
+	'‟': `"`,
+	'ˮ': `"`,
+	'„': `"`,
+	'‹': `"`,
+	'›': `"`,
+	'❮': `"`,
+	'❯': '"',
+	'«': `"`,
+	'»': `"`,
+	'״': `"`,
+	'❝': `"`,
+	'❞': `"`,
+	'🙶': `"`,
+	'🙷': `"`,
+	'⹂': `"`,
+	'〝': `"`,
+	'〞': `"`,
+	'〟': `"`,
+	'＂': `"`,
+	'❠': `"`,
+	'🙸': `"`,
+
+	'ߵ': `'`,
+	'ߴ': `'`,
+	'’': `'`,
+	'‘': `'`,
+	'ʹ': `'`,
+	'ʼ': `'`,
+	'＇': `'`,
+	'ʻ': `'`,
+	'՚': `'`,
+	'՛': `'`,
+	'❛': `'`,
+	'❜': `'`,
+	'❟': `'`,
+
+	'，': `,`,
+	'、': `,`,
+
+	'：': `:`,
+
+	'；': `;`,
+
+	'。': `.`,
+
+	'？': `?`,
+	'؟': `?`,
+
+	'！': `!`,
+	'¡': `!`,
+}

package/src/recognition/WhisperSTT.ts

@@ -2,7 +2,7 @@ import type * as Onnx from 'onnxruntime-node'
 
 import { Logger } from '../utilities/Logger.js'
 import { computeMelSpectogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectogram.js'
-import { clip, containsInvalidCodepoint, getIntegerRange, getTokenRepetitionScore, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
+import { clip, getIntegerRange, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
 import { indexOfMax, logOfVector, logSumExp, meanOfVector, softmax, stdDeviationOfVector } from '../math/VectorMath.js'
 
 import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
@@ -22,8 +22,9 @@ import { type Tiktoken } from 'tiktoken/lite'
 import { isPunctuation, isWhitespace, isWord, splitToSentences, splitToWords } from '../nlp/Segmentation.js'
 import { medianOf5Filter } from '../math/MedianFilter.js'
 import { getDeflateCompressionMetricsForString } from '../utilities/Compression.js'
-import { getOnnxSessionOptions, makeOnnxLikeFloat32Tensor, OnnxExecutionProvider, OnnxLikeFloat32Tensor } from '../utilities/OnnxUtilities.js'
+import { dmlProviderAvailable, getOnnxSessionOptions, makeOnnxLikeFloat32Tensor, OnnxExecutionProvider, OnnxLikeFloat32Tensor } from '../utilities/OnnxUtilities.js'
 import { murmurHash3_int32Input } from '../utilities/Hashing.js'
+import { containsInvalidCodepoint, getTokenRepetitionScore } from '../utilities/StringUtilities.js'
 
 export async function recognize(
 	sourceRawAudio: RawAudio,
@@ -1841,17 +1842,17 @@ export function isEnglishOnlyModel(modelName: WhisperModelName) {
 	return modelName.endsWith('.en')
 }
 
-function getDefaultEncoderProvidersForModel(modelName: WhisperModelName): OnnxExecutionProvider[] {
-	if (process.platform === 'win32') {
+export function getDefaultEncoderProvidersForModel(modelName: WhisperModelName): OnnxExecutionProvider[] {
+	if (dmlProviderAvailable()) {
 		return ['dml', 'cpu']
 	} else {
 		return []
 	}
 }
 
-function getDefaultDecoderProvidersForModel(modelName: WhisperModelName): OnnxExecutionProvider[] {
+export function getDefaultDecoderProvidersForModel(modelName: WhisperModelName): OnnxExecutionProvider[] {
 	if (modelName.startsWith('small') || modelName.startsWith('medium') || modelName.startsWith('large')) {
-		if (process.platform === 'win32') {
+		if (dmlProviderAvailable()) {
 			return ['dml', 'cpu']
 		} else {
 			return []