echogarden 1.3.0 → 1.3.2

package/src/alignment/TextAlignment.ts

@@ -0,0 +1,96 @@
+import { type PreTrainedModel, type PreTrainedTokenizer } from '@echogarden/transformers-nodejs-lite'
+import { Logger } from '../utilities/Logger.js'
+import { loadPackage } from '../utilities/PackageManager.js'
+
+export async function alignText(text1: string, text2: string) {
+	const logger = new Logger()
+
+	text1 = text1.replaceAll(/(\r?\n)+/g, ' ')
+	text2 = text2.replaceAll(/(\r?\n)+/g, ' ')
+
+	const modelPath = await loadPackage(`xenova-multilingual-e5-small-quantized`)
+
+	const embeddingModel = new E5TextEmbedding(modelPath)
+
+	logger.start(`Initialize E5 embedding model`)
+	await embeddingModel.initializeIfNeeded()
+
+	logger.start(`Tokenize text 1`)
+	const inputs1 = await embeddingModel.tokenizeToModelInputs(text1)
+
+	logger.start(`Infer embeddings for text 1`)
+	const embeddings1 = await embeddingModel.inferTokenEmbeddings(inputs1)
+
+	logger.start(`Tokenize text 2`)
+	const inputs2 = await embeddingModel.tokenizeToModelInputs(text2)
+
+	logger.start(`Infer embeddings for text 2`)
+	const embeddings2 = await embeddingModel.inferTokenEmbeddings(inputs1)
+
+	logger.end()
+
+	logger.log(embeddings1)
+}
+
+export class E5TextEmbedding {
+	tokenizer?: PreTrainedTokenizer
+	model?: PreTrainedModel
+
+	constructor(public readonly modelPath: string) {
+	}
+
+	async tokenizeToModelInputs(text: string) {
+		await this.initializeIfNeeded()
+
+		const inputs = await this.tokenizer!(text)
+
+		return inputs
+	}
+
+	async inferTokenEmbeddings(inputs: any) {
+		await this.initializeIfNeeded()
+
+		const tokensText = this.tokenizer!.model.convert_ids_to_tokens(Array.from(inputs.input_ids.data))
+
+		const result = await this.model!(inputs)
+
+		const lastHiddenState = result.last_hidden_state
+
+		const tokenCount = lastHiddenState.dims[1]
+		const embeddingSize = lastHiddenState.dims[2]
+
+		const tokenEmbeddings: TokenEmbeddingData[] = []
+
+		for (let i = 0; i < tokenCount; i++) {
+			const tokenEmbeddingVector = lastHiddenState.data.slice(i * embeddingSize, (i + 1) * embeddingSize)
+
+			const tokenId = Number(inputs.input_ids.data[i])
+			const tokenText = tokensText[i]
+
+			tokenEmbeddings.push({
+				id: tokenId,
+				text: tokenText,
+				embeddingVector: tokenEmbeddingVector
+			})
+		}
+
+		return tokenEmbeddings
+	}
+
+	async initializeIfNeeded() {
+		if (this.tokenizer && this.model) {
+			return
+		}
+
+		const { AutoTokenizer, AutoModel } = await import('@echogarden/transformers-nodejs-lite')
+
+		this.tokenizer = await AutoTokenizer.from_pretrained(this.modelPath)
+		this.model = await AutoModel.from_pretrained(this.modelPath)
+	}
+}
+
+export interface TokenEmbeddingData {
+	id: number
+	text: string
+	embeddingVector: Float32Array
+}

package/src/api/Alignment.ts

@@ -13,10 +13,15 @@ import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignmen
 import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 import { type EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
 import { isWord } from '../nlp/Segmentation.js'
+import { alignText } from '../alignment/TextAlignment.js'
+import { translateText } from '../text-translation/NLLBTextTranslation.js'
 
 const log = logToStderr
 
 export async function align(input: AudioSourceParam, transcript: string, options: AlignmentOptions): Promise<AlignmentResult> {
+	//await alignText(transcript, transcript)
+	//await translateText(transcript, 'en', 'de')
+
 	const logger = new Logger()
 
 	const startTimestamp = logger.getTimestamp()
@@ -38,8 +43,10 @@ export async function align(input: AudioSourceParam, transcript: string, options
 		logger.end()
 		logger.log(``)
 
+		logger.start(`Resample audio to 16kHz mono`)
 		sourceRawAudio = await ensureRawAudio(isolatedRawAudio, 16000, 1)
 	} else {
+		logger.start(`Resample audio to 16kHz mono`)
 		sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
 	}
 
@@ -52,7 +59,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
 		logger.end()
 	}
 
-	logger.start('Prepare for alignment')
+	logger.start('Normalize and trim audio')
 
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
@@ -69,6 +76,8 @@ export async function align(input: AudioSourceParam, transcript: string, options
 		}
 	}
 
+	logger.end()
+
 	let language: string
 
 	if (options.language) {

package/src/api/Denoising.ts

@@ -14,8 +14,6 @@ export async function denoise(input: AudioSourceParam, options: DenoisingOptions
 	const logger = new Logger()
 	const startTime = logger.getTimestamp()
 
-	logger.start('Prepare for denoising')
-
 	options = extendDeep(defaultDenoisingOptions, options)
 
 	const inputRawAudio = await ensureRawAudio(input)

package/src/api/LanguageDetection.ts

@@ -28,6 +28,7 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 
 	const inputRawAudio = await ensureRawAudio(input)
 
+	logger.start(`Resample audio to 16kHz mono`)
 	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
@@ -39,13 +40,11 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 		logger.end()
 	}
 
-	logger.start('Prepare for speech language detection')
+	logger.start(`Initialize ${options.engine} module`)
 
 	const defaultLanguage = options.defaultLanguage!
 	const fallbackThresholdProbability = options.fallbackThresholdProbability!
 
-	logger.start(`Initialize ${options.engine} module`)
-
 	let detectedLanguageProbabilities: LanguageDetectionResults
 
 	switch (options.engine) {

package/src/api/Recognition.ts

@@ -41,8 +41,10 @@ export async function recognize(input: AudioSourceParam, options: RecognitionOpt
 		logger.end()
 		logger.log(``)
 
+		logger.start(`Resample audio to 16kHz mono`)
 		sourceRawAudio = await ensureRawAudio(isolatedRawAudio, 16000, 1)
 	} else {
+		logger.start(`Resample audio to 16kHz mono`)
 		sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
 	}
 
@@ -55,7 +57,7 @@ export async function recognize(input: AudioSourceParam, options: RecognitionOpt
 		logger.end()
 	}
 
-	logger.start('Prepare for recognition')
+	logger.start('Normalize and trim audio')
 
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])

package/src/api/Synthesis.ts

@@ -301,7 +301,7 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 	const startTimestamp = logger.getTimestamp()
 
-	logger.start('Prepare for synthesis')
+	logger.start('Prepare text for synthesis')
 
 	const simplifiedText = simplifyPunctuationCharacters(text)
 

package/src/api/Translation.ts

@@ -43,8 +43,10 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
 		logger.end()
 		logger.log(``)
 
+		logger.start(`Resample audio to 16kHz mono`)
 		sourceRawAudio = await ensureRawAudio(isolatedRawAudio, 16000, 1)
 	} else {
+		logger.start(`Resample audio to 16kHz mono`)
 		sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
 	}
 
@@ -57,7 +59,7 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
 		logger.end()
 	}
 
-	logger.start('Prepare for speech translation')
+	logger.start('Normalize and trim audio')
 
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])

package/src/api/TranslationAlignment.ts

@@ -35,8 +35,10 @@ export async function alignTranslation(input: AudioSourceParam, transcript: stri
 		logger.end()
 		logger.log(``)
 
+		logger.start(`Resample audio to 16kHz mono`)
 		sourceRawAudio = await ensureRawAudio(isolatedRawAudio, 16000, 1)
 	} else {
+		logger.start(`Resample audio to 16kHz mono`)
 		sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
 	}
 
@@ -49,11 +51,13 @@ export async function alignTranslation(input: AudioSourceParam, transcript: stri
 		logger.end()
 	}
 
-	logger.start('Prepare for alignment')
+	logger.start('Normalize and trim audio')
 
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
 
+	logger.end()
+
 	let sourceLanguage: string
 
 	if (options.sourceLanguage) {

package/src/api/VoiceActivityDetection.ts

@@ -20,16 +20,15 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 
 	const startTimestamp = logger.getTimestamp()
 
-	logger.start('Prepare for voice activity detection')
-
 	const inputRawAudio = await ensureRawAudio(input)
 
+	logger.start(`Resample audio to 16kHz mono`)
 	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
 
-	options = extendDeep(defaultVADOptions, options)
-
 	logger.start(`Detect voice activity with ${options.engine}`)
 
+	options = extendDeep(defaultVADOptions, options)
+
 	const activityThreshold = options.activityThreshold!
 
 	let verboseTimeline: Timeline
@@ -224,6 +223,10 @@ export function convertCroppedToUncroppedTimeline(timeline: Timeline, uncropTime
 function mapUsingUncropTimeline(startTimeInCroppedAudio: number, endTimeInCroppedAudio: number, uncropTimeline: Timeline) {
 	let offsetInCroppedAudio = 0
 
+	if (endTimeInCroppedAudio < startTimeInCroppedAudio) {
+		endTimeInCroppedAudio = startTimeInCroppedAudio
+	}
+
 	let bestOverlapDuration = -1
 	let mappedStartTime = -1
 	let mappedEndTime = -1

package/src/audio/AudioUtilities.ts

@@ -11,8 +11,8 @@ export function encodeRawAudioToWave(rawAudio: RawAudio, bitDepth: BitDepth = 16
 	return encodeWave(rawAudio, bitDepth, sampleFormat, speakerPositionMask)
 }
 
-export function decodeWaveToRawAudio(waveFileBuffer: Buffer, ignoreTruncatedChunks = false) {
-	return decodeWave(waveFileBuffer, ignoreTruncatedChunks)
+export function decodeWaveToRawAudio(waveFileBuffer: Buffer, ignoreTruncatedChunks = true, ignoreOverflowingDataChunks = true) {
+	return decodeWave(waveFileBuffer, ignoreTruncatedChunks, ignoreOverflowingDataChunks)
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////

package/src/cli/CLILauncher.ts

@@ -12,9 +12,8 @@ const scriptArgs = process.argv.slice(2)
 const cliScriptPath = resolveToModuleRootDir('dist/cli/CLIStarter.js')
 
 const args = [
-	'--no-warnings',
-	'--no-experimental-fetch',
 	'--experimental-wasi-unstable-preview1',
+	'--no-warnings',
 	cliScriptPath,
 	...scriptArgs
 ]

package/src/codecs/FFMpegTranscoder.ts

@@ -37,7 +37,11 @@ export async function decodeToChannels(input: string | Buffer, outSampleRate?: n
 
 	const waveAudio = await transcode(input, outputOptions)
 
-	const { rawAudio } = decodeWaveToRawAudio(waveAudio, true)
+	const logger = new Logger()
+
+	logger.start(`Convert wave buffer to raw audio`)
+	const { rawAudio } = decodeWaveToRawAudio(waveAudio)
+	logger.end()
 
 	return rawAudio
 }

package/src/codecs/WaveCodec.ts

@@ -18,18 +18,20 @@ export function encodeWave(rawAudio: RawAudio, bitDepth: BitDepth = 16, sampleFo
 
 	const dataSubChunkBuffer = Buffer.alloc(4 + 4 + audioDataLength)
 	dataSubChunkBuffer.write('data', 0, 'ascii')
-	dataSubChunkBuffer.writeUint32LE(audioDataLength, 4)
+	const dataChunkLength = Math.min(audioDataLength, 4294967295) // Ensure large data chunk length is clipped to max
+	dataSubChunkBuffer.writeUint32LE(dataChunkLength, 4)
 	dataSubChunkBuffer.set(audioBuffer, 8)
 
 	const riffChunkHeaderBuffer = Buffer.alloc(12)
 	riffChunkHeaderBuffer.write('RIFF', 0, 'ascii')
-	riffChunkHeaderBuffer.writeUint32LE(4 + formatSubChunkBuffer.length + dataSubChunkBuffer.length, 4)
+	const riffChunkLength = Math.min(4 + formatSubChunkBuffer.length + dataSubChunkBuffer.length, 4294967295) // Ensure large RIFF chunk length is clipped to max
+	riffChunkHeaderBuffer.writeUint32LE(riffChunkLength, 4)
 	riffChunkHeaderBuffer.write('WAVE', 8, 'ascii')
 
 	return Buffer.concat([riffChunkHeaderBuffer, formatSubChunkBuffer, dataSubChunkBuffer])
 }
 
-export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = false) {
+export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = true, ignoreOverflowingDataChunks = true) {
 	let readOffset = 0
 
 	const riffId = waveData.subarray(readOffset, readOffset + 4).toString('ascii')
@@ -40,7 +42,7 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = false) {
 
 	readOffset += 4
 
-	const riffChunkSize = waveData.readUInt32LE(readOffset)
+	let riffChunkSize = waveData.readUInt32LE(readOffset)
 
 	readOffset += 4
 
@@ -50,6 +52,10 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = false) {
 		throw new Error('Not a valid wave file. No WAVE id found at offset 8.')
 	}
 
+	if (ignoreOverflowingDataChunks && riffChunkSize === 4294967295) {
+		riffChunkSize = waveData.length - 8
+	}
+
 	if (riffChunkSize < waveData.length - 8) {
 		throw new Error(`RIFF chunk length ${riffChunkSize} is smaller than the remaining size of the buffer (${waveData.length - 8})`)
 	}
@@ -67,7 +73,7 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = false) {
 		const subChunkIdentifier = waveData.subarray(readOffset, readOffset + 4).toString('ascii')
 		readOffset += 4
 
-		const subChunkSize = waveData.readUInt32LE(readOffset)
+		let subChunkSize = waveData.readUInt32LE(readOffset)
 		readOffset += 4
 
 		if (!ignoreTruncatedChunks && subChunkSize > waveData.length - readOffset) {
@@ -81,6 +87,10 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = false) {
 				throw new Error('A data subchunk was encountered before a format subchunk')
 			}
 
+			if (ignoreOverflowingDataChunks && subChunkSize === 4294967295) {
+				subChunkSize = waveData.length - readOffset
+			}
+
 			// If the data chunk is truncated, but truncations are ignored,
 			// it would be read up to the end of the buffer
 			dataBuffers.push(waveData.subarray(readOffset, readOffset + subChunkSize))

package/src/dsp/SpeexResampler.ts

@@ -1,16 +1,27 @@
 import { RawAudio, cloneRawAudio } from '../audio/AudioUtilities.js'
+import { concatFloat32Arrays } from '../utilities/Utilities.js'
 import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js'
 
 let speexResamplerInstance: any
 
-export async function resampleAudioSpeex(rawAudio: RawAudio, outSampleRate: number, quality = 0) {
+export async function resampleAudioSpeex(rawAudio: RawAudio, outSampleRate: number, quality = 0): Promise<RawAudio> {
 	const channelCount = rawAudio.audioChannels.length
 	const inSampleRate = rawAudio.sampleRate
 
-	if (inSampleRate == outSampleRate) {
+	const totalSampleCount = rawAudio.audioChannels[0].length
+	const sampleRateRatio = outSampleRate / inSampleRate
+
+	if (inSampleRate === outSampleRate) {
 		return cloneRawAudio(rawAudio)
 	}
 
+	if (totalSampleCount === 0) {
+		return {
+			...cloneRawAudio(rawAudio),
+			sampleRate: outSampleRate
+		} as RawAudio
+	}
+
 	const m = await getSpeexResamplerInstance()
 	const wasmMemory = new WasmMemoryManager(m)
 
@@ -23,48 +34,76 @@ export async function resampleAudioSpeex(rawAudio: RawAudio, outSampleRate: numb
 	}
 
 	const initErrRef = wasmMemory.allocInt32()
-	const resamplerState = m._speex_resampler_init(channelCount, inSampleRate, outSampleRate, quality, initErrRef.address)
+	const resamplerStateAddress = m._speex_resampler_init(channelCount, inSampleRate, outSampleRate, quality, initErrRef.address)
 	let resultCode = initErrRef.value
 
 	if (resultCode != 0) {
 		throw new Error(`Speex resampler failed while initializing with code ${resultCode}: ${speexResultCodeToString(resultCode)}`)
 	}
 
-	const resampledAudio: RawAudio = { audioChannels: [], sampleRate: outSampleRate }
+	const inputLatency = m._speex_resampler_get_input_latency(resamplerStateAddress)
+	const outputLatency = m._speex_resampler_get_output_latency(resamplerStateAddress)
+
+	const maxChunkSize = 2 ** 20
+
+	const inputChunkSampleCountRef = wasmMemory.allocInt32()
+	const outputChunkSampleCountRef = wasmMemory.allocInt32()
+
+	const inputChunkSamplesRef = wasmMemory.allocFloat32Array(maxChunkSize * 2)
+	const outputChunkSamplesRef = wasmMemory.allocFloat32Array(Math.floor(maxChunkSize * sampleRateRatio) * 2)
+
+	const resampledAudioChunksForChannels: Float32Array[][] = []
+
+	for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) {
+		resampledAudioChunksForChannels.push([])
+	}
 
 	for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) {
-		const channelSamples = rawAudio.audioChannels[channelIndex]
+		for (let readOffset = 0; readOffset < totalSampleCount;) {
+			const isLastChunk = readOffset + maxChunkSize >= totalSampleCount
 
-		const inputLatency = m._speex_resampler_get_input_latency(resamplerState)
-		const outputLatency = m._speex_resampler_get_output_latency(resamplerState)
+			const inputPaddingSize = isLastChunk ? inputLatency : 0
+			const maxSamplesToRead = Math.min(maxChunkSize, totalSampleCount - readOffset) + inputPaddingSize
 
-		const inSampleCount = channelSamples.length
-		const inSampleCountRef = wasmMemory.allocInt32()
-		inSampleCountRef.value = inSampleCount + inputLatency
+			const maxSamplesToWrite = outputChunkSamplesRef.length
 
-		const inSamplesRef = wasmMemory.allocFloat32Array(inSampleCountRef.value)
-		inSamplesRef.view.set(channelSamples)
+			const inputChunkSamplesForChannel = rawAudio.audioChannels[channelIndex].slice(readOffset, readOffset + maxSamplesToRead)
 
-		const outSampleCount = Math.floor((inSampleCount / inSampleRate) * outSampleRate)
-		const outSampleCountRef = wasmMemory.allocInt32()
-		outSampleCountRef.value = outSampleCount + outputLatency
+			inputChunkSampleCountRef.value = maxSamplesToRead
+			outputChunkSampleCountRef.value = maxSamplesToWrite
 
-		const outSamplesRef = wasmMemory.allocFloat32Array(outSampleCountRef.value)
+			inputChunkSamplesRef.view.set(inputChunkSamplesForChannel)
+			resultCode = m._speex_resampler_process_float(resamplerStateAddress, channelIndex, inputChunkSamplesRef.address, inputChunkSampleCountRef.address, outputChunkSamplesRef.address, outputChunkSampleCountRef.address)
 
-		resultCode = m._speex_resampler_process_float(resamplerState, channelIndex, inSamplesRef.address, inSampleCountRef.address, outSamplesRef.address, outSampleCountRef.address)
+			if (resultCode != 0) {
+				throw new Error(`Speex resampler failed while resampling with code ${resultCode}: ${speexResultCodeToString(resultCode)}`)
+			}
 
-		if (resultCode != 0) {
-			throw new Error(`Speex resampler failed while resampling with code ${resultCode}: ${speexResultCodeToString(resultCode)}`)
-		}
+			const samplesReadCount = inputChunkSampleCountRef.value
+			const samplesWrittenCount = outputChunkSampleCountRef.value
+
+			const resampledChannelAudio = outputChunkSamplesRef.view.slice(0, samplesWrittenCount)
 
-		const resampledChannelAudio = outSamplesRef.view.slice(outputLatency)
+			resampledAudioChunksForChannels[channelIndex].push(resampledChannelAudio)
 
-		resampledAudio.audioChannels.push(resampledChannelAudio)
+			readOffset += samplesReadCount
+		}
 	}
 
-	m._speex_resampler_destroy(resamplerState)
+	m._speex_resampler_destroy(resamplerStateAddress)
 	wasmMemory.freeAll()
 
+	const resampledAudio: RawAudio = {
+		audioChannels: [],
+		sampleRate: outSampleRate
+	}
+
+	for (let i = 0; i < channelCount; i++) {
+		resampledAudioChunksForChannels[i][0] = resampledAudioChunksForChannels[i][0].slice(outputLatency)
+
+		resampledAudio.audioChannels.push(concatFloat32Arrays(resampledAudioChunksForChannels[i]))
+	}
+
 	return resampledAudio
 }
 

package/src/synthesis/CoquiServerTTS.ts

@@ -21,7 +21,7 @@ export async function synthesize(text: string, speakerId: string | null, serverU
 
 	const waveData = Buffer.from(response.data)
 
-	const rawAudio = decodeWaveToRawAudio(waveData).rawAudio
+	const { rawAudio } = decodeWaveToRawAudio(waveData)
 
 	logger.end()
 

package/src/synthesis/VitsTTS.ts

@@ -52,7 +52,7 @@ export class VitsTTS {
 
 		await this.initializeIfNeeded()
 
-		await logger.startAsync('Prepare for synthesis')
+		await logger.startAsync('Prepare for VITS synthesis')
 
 		const metadata = this.metadata
 		const phonemeMap = this.phonemeMap!
@@ -78,7 +78,7 @@ export class VitsTTS {
 			voice: espeakVoice,
 			useKlatt: false
 		}
-		
+
 		const { referenceSynthesizedAudio, referenceTimeline, fragments, phonemizedFragmentsSubstitutions, phonemizedSentence } = await Espeak.preprocessAndSynthesize(sentence, languageCode, espeakOptions, lexicons)
 
 		if (phonemizedSentence.length == 0) {