echogarden 1.3.3 → 1.4.0

package/src/alignment/DTWSequenceAlignmentWindowed.ts

@@ -1,5 +1,3 @@
-import chalk from 'chalk'
-import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { AlignmentPath } from './SpeechAlignment.js'
 
@@ -11,7 +9,10 @@ export function alignDTWWindowed<T, U>(sequence1: T[], sequence2: U[], costFunct
 	}
 
 	if (sequence1.length == 0 || sequence2.length == 0) {
-		return { path: [] as AlignmentPath, pathCost: 0 }
+		return {
+			path: [] as AlignmentPath,
+			pathCost: 0
+		}
 	}
 
 	// Compute accumulated cost matrix (transposed)
@@ -38,10 +39,10 @@ function computeAccumulatedCostMatrixTransposed<T, U>(sequence1: T[], sequence2:
 
 	const accumulatedCostMatrixTransposed: Float32Array[] = new Array<Float32Array>(columnCount)
 
-	// Initialize window start offsets array
+	// Initialize an array to store window start offsets
 	const windowStartOffsets = new Int32Array(columnCount)
 
-	// Compute matrix column by column
+	// Compute accumulated cost matrix column by column
 	for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) {
 		// Create new column and add it to the matrix
 		const currentColumn = new Float32Array(rowCount)
@@ -65,13 +66,13 @@ function computeAccumulatedCostMatrixTransposed<T, U>(sequence1: T[], sequence2:
 			windowStartOffset = windowEndOffset - rowCount
 		}
 
-		// Store the start offset in the array
+		// Store the start offset for this column
 		windowStartOffsets[columnIndex] = windowStartOffset
 
 		// Get target sequence1 value
 		const targetSequence1Value = sequence1[columnIndex]
 
-		// If first column, fill it only using the 'up' neighbor
+		// If this is the first column, fill it only using the 'up' neighbors
 		if (columnIndex == 0) {
 			for (let rowIndex = 1; rowIndex < rowCount; rowIndex++) {
 				const cost = costFunction(targetSequence1Value, sequence2[windowStartOffset + rowIndex])
@@ -84,11 +85,15 @@ function computeAccumulatedCostMatrixTransposed<T, U>(sequence1: T[], sequence2:
 		}
 
 		// If not first column
+
+		// Store the column to the left
 		const leftColumn = accumulatedCostMatrixTransposed[columnIndex - 1]
 
-		// Compute the delta between the current window offset and previous column's window offset
+		// Compute the delta between the current window start offset
+		// and left column's window offset
 		const windowOffsetDelta = windowStartOffset - windowStartOffsets[columnIndex - 1]
 
+		// Iterate over all rows in the window
 		for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
 			// Compute the cost for current cell
 			const cost = costFunction(targetSequence1Value, sequence2[windowStartOffset + rowIndex])
@@ -115,12 +120,25 @@ function computeAccumulatedCostMatrixTransposed<T, U>(sequence1: T[], sequence2:
 				upAndLeftCost = leftColumn[upAndLeftRowIndex]
 			}
 
+			// Find the minimum of all neighbors
+			let minimumNeighborCost = minimumOf3(upCost, leftCost, upAndLeftCost)
+
+			// If all neighbors are infinity, then it means there is a "jump" between the window
+			// of the current column and the left column, and they don't have overlapping rows.
+			// In this case, only the cost of the current cell will be used
+			if (minimumNeighborCost === Infinity) {
+				minimumNeighborCost = 0
+			}
+
 			// Write cost + minimum neighbor cost to the current column
-			currentColumn[rowIndex] = cost + minimumOf3(upCost, leftCost, upAndLeftCost)
+			currentColumn[rowIndex] = cost + minimumNeighborCost
 		}
 	}
 
-	return { accumulatedCostMatrixTransposed, windowStartOffsets }
+	return {
+		accumulatedCostMatrixTransposed,
+		windowStartOffsets
+	}
 }
 
 function computeBestPathTransposed(accumulatedCostMatrixTransposed: Float32Array[], windowStartOffsets: Int32Array) {
@@ -129,6 +147,8 @@ function computeBestPathTransposed(accumulatedCostMatrixTransposed: Float32Array
 
 	const bestPath: AlignmentPath = []
 
+	// Start at the bottom right corner and find the best path
+	// towards the top left
 	let columnIndex = columnCount - 1
 	let rowIndex = rowCount - 1
 
@@ -136,19 +156,21 @@ function computeBestPathTransposed(accumulatedCostMatrixTransposed: Float32Array
 		const windowStartIndex = windowStartOffsets[columnIndex]
 		const windowStartDelta = columnIndex > 0 ? windowStartIndex - windowStartOffsets[columnIndex - 1] : 0
 
+		// Add the current cell to the best path
 		bestPath.push({
 			source: columnIndex,
 			dest: windowStartIndex + rowIndex
 		})
 
+		// Retrieve the cost for the 'up' (insertion) neighbor
 		const upRowIndex = rowIndex - 1
-		const upColumnIndex = columnIndex
 		let upCost = Infinity
 
 		if (upRowIndex >= 0) {
-			upCost = accumulatedCostMatrixTransposed[upColumnIndex][upRowIndex] // insertion
+			upCost = accumulatedCostMatrixTransposed[columnIndex][upRowIndex] // insertion
 		}
 
+		// Retrieve the cost for the 'left' (deletion) neighbor
 		const leftRowIndex = rowIndex + windowStartDelta
 		const leftColumnIndex = columnIndex - 1
 		let leftCost = Infinity
@@ -157,6 +179,7 @@ function computeBestPathTransposed(accumulatedCostMatrixTransposed: Float32Array
 			leftCost = accumulatedCostMatrixTransposed[leftColumnIndex][leftRowIndex] // deletion
 		}
 
+		// Retrieve the cost for the 'up and left' (match) neighbor
 		const upAndLeftRowIndex = rowIndex - 1 + windowStartDelta
 		const upAndLeftColumnIndex = columnIndex - 1
 		let upAndLeftCost = Infinity
@@ -165,25 +188,42 @@ function computeBestPathTransposed(accumulatedCostMatrixTransposed: Float32Array
 			upAndLeftCost = accumulatedCostMatrixTransposed[upAndLeftColumnIndex][upAndLeftRowIndex] // match
 		}
 
+		// If all neighbors have a cost of infinity, it means
+		// there is a "jump" between the window for the current and previous column
 		if (upCost == Infinity && leftCost == Infinity && upAndLeftCost == Infinity) {
-			const logger = new Logger()
-
-			logger.setAsActiveLogger()
-			logger.logTitledMessage(`computeBestPath`, `unexpected - all cost directions are equal to infinity (${columnIndex}, ${rowIndex}).`, chalk.yellowBright, 'warning')
-			logger.unsetAsActiveLogger()
-		}
-
-		const smallestCostDirection = argIndexOfMinimumOf3(upCost, leftCost, upAndLeftCost)
-
-		if (smallestCostDirection == 1) {
-			rowIndex = upRowIndex
-			columnIndex = upColumnIndex
-		} else if (smallestCostDirection == 2) {
-			rowIndex = leftRowIndex
-			columnIndex = leftColumnIndex
+			// In that case:
+			//
+			// If there are rows above
+			if (upRowIndex >= 0) {
+				// Move upward
+				rowIndex = upRowIndex
+			} else if (leftColumnIndex >= 0) {
+				// Otherwise, move to the left
+				columnIndex = leftColumnIndex
+			} else {
+				// Since we know that either columnIndex > 0 or rowIndex > 0,
+				// one of these directions must be available.
+				// This error should never happen
+
+				throw new Error(`Unexpected state: columnIndex: ${columnIndex}, rowIndex: ${rowIndex}`)
+			}
 		} else {
-			rowIndex = upAndLeftRowIndex
-			columnIndex = upAndLeftColumnIndex
+			// Choose the direction with the smallest cost
+			const smallestCostDirection = argIndexOfMinimumOf3(upCost, leftCost, upAndLeftCost)
+
+			if (smallestCostDirection == 1) {
+				// Move upward
+				rowIndex = upRowIndex
+				// The upper column index stays the same
+			} else if (smallestCostDirection == 2) {
+				// Move to the left
+				rowIndex = leftRowIndex
+				columnIndex = leftColumnIndex
+			} else {
+				// Move upward and to the left
+				rowIndex = upAndLeftRowIndex
+				columnIndex = upAndLeftColumnIndex
+			}
 		}
 	}
 

package/src/alignment/SpeechAlignment.ts

@@ -38,12 +38,12 @@ export async function alignUsingDtw(
 	let relativeCenters: number[] | undefined
 
 	for (let passIndex = 0; passIndex < windowDurations.length; passIndex++) {
+		const granularity = granularities[passIndex]
 		const windowDuration = windowDurations[passIndex]
-		const granularity = resolveAutoGranularityIfNeeded(granularities[passIndex], rawAudioDuration)
 
 		logger.logTitledMessage(`\nStarting alignment pass ${passIndex + 1}/${windowDurations.length}`, `max window duration: ${windowDuration}s, granularity: ${granularity}`, chalk.magentaBright)
 
-		const mfccOptions = extendDefaultMfccOptions({ ...getMfccOptionsForGranularity(granularity, rawAudioDuration), zeroFirstCoefficient: true }) as MfccOptions
+		const mfccOptions = extendDefaultMfccOptions({ ...getMfccOptionsForGranularity(granularity), zeroFirstCoefficient: true }) as MfccOptions
 
 		framesPerSecond = 1 / mfccOptions.hopDuration!
 
@@ -67,7 +67,7 @@ export async function alignUsingDtw(
 			}
 		}
 
-		logger.start('Align MFCC features using DTW')
+		logger.start('Align reference and source MFCC features using DTW')
 		const dtwWindowLength = Math.floor(windowDuration * framesPerSecond)
 
 		let centerIndexes: number[] | undefined
@@ -458,7 +458,7 @@ export async function createAlignmentReferenceUsingEspeakForFragments(fragments:
 	progressLogger.start("Load espeak module")
 	const Espeak = await import("../synthesis/EspeakTTS.js")
 
-	progressLogger.start("Create alignment reference with eSpeak")
+	progressLogger.start("Synthesize alignment reference with eSpeak")
 
 	const result = await Espeak.synthesizeFragments(fragments, espeakOptions)
 
@@ -476,7 +476,7 @@ export async function createAlignmentReferenceUsingEspeakForFragments(fragments:
 export async function createAlignmentReferenceUsingEspeak(transcript: string, language: string, plaintextOptions?: API.PlainTextOptions, customLexiconPaths?: string[], insertSeparators?: boolean) {
 	const logger = new Logger()
 
-	logger.start('Create alignment reference with eSpeak')
+	logger.start('Synthesize alignment reference with eSpeak')
 
 	const synthesisOptions: API.SynthesisOptions = {
 		engine: 'espeak',
@@ -544,25 +544,9 @@ function getMappedFrameIndexForPath(referenceFrameIndex: number, compactedPath:
 	return mappedFrameIndex
 }
 
-function resolveAutoGranularityIfNeeded(granularity: DtwGranularity, audioDuration: number) {
-	if (granularity != 'auto') {
-		return granularity
-	}
-
-	if (audioDuration < 60) {
-		return 'high'
-	} else if (audioDuration < 60 * 10) {
-		return 'medium'
-	} else {
-		return 'low'
-	}
-}
-
-function getMfccOptionsForGranularity(granularity: DtwGranularity, audioDuration: number) {
+function getMfccOptionsForGranularity(granularity: DtwGranularity) {
 	let mfccOptions: MfccOptions
 
-	granularity = resolveAutoGranularityIfNeeded(granularity, audioDuration)
-
 	if (granularity == 'xx-low') {
 		mfccOptions = { windowDuration: 0.400, hopDuration: 0.160, fftOrder: 8192 }
 	} else if (granularity == 'x-low') {
@@ -595,4 +579,4 @@ export type CompactedPathEntry = {
 	first: number, last: number
 }
 
-export type DtwGranularity = 'auto' | 'xx-low' | 'x-low' | 'low' | 'medium' | 'high' | 'x-high'
+export type DtwGranularity = 'xx-low' | 'x-low' | 'low' | 'medium' | 'high' | 'x-high'

package/src/api/Alignment.ts

@@ -64,18 +64,6 @@ export async function align(input: AudioSourceParam, transcript: string, options
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
 
-	if (options.dtw!.windowDuration == null) {
-		const sourceAudioDuration = getRawAudioDuration(sourceRawAudio)
-
-		if (sourceAudioDuration < 5 * 60) { // If up to 5 minutes, set window to one minute
-			options.dtw!.windowDuration = 60
-		} else if (sourceAudioDuration < 60 * 60) { // If up to 1 hour, set window to 20% of total duration
-			options.dtw!.windowDuration = Math.ceil(sourceAudioDuration * 0.2)
-		} else { // If 1 hour or more, set window to 12 minutes
-			options.dtw!.windowDuration = 12 * 60
-		}
-	}
-
 	logger.end()
 
 	let language: string
@@ -87,7 +75,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 		logger.logTitledMessage('Language specified', formatLanguageCodeWithName(language))
 	} else {
-		logger.start('No language specified. Detecting language')
+		logger.start('No language specified. Detect language')
 		const { detectedLanguage } = await API.detectTextLanguage(transcript, options.languageDetection || {})
 
 		language = detectedLanguage
@@ -103,7 +91,9 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 	const { alignUsingDtwWithRecognition, alignUsingDtw } = await import('../alignment/SpeechAlignment.js')
 
-	function getDtwWindowDurationsAndGranularities() {
+	function getDtwWindowGranularitiesAndDurations() {
+		const sourceAudioDuration = getRawAudioDuration(sourceRawAudio)
+
 		let granularities: DtwGranularity[]
 		let windowDurations: number[]
 
@@ -112,25 +102,52 @@ export async function align(input: AudioSourceParam, transcript: string, options
 		} else if (Array.isArray(options.dtw!.granularity)) {
 			granularities = options.dtw!.granularity
 		} else {
-			granularities = ['auto']
+			if (sourceAudioDuration < 1 * 60) {
+				// If up to 1 minute, set granularity to high, single pass
+				granularities = ['high']
+			} else if (sourceAudioDuration < 5 * 60) {
+				// If up to 5 minutes, set granularity to medium, single pass
+				granularities = ['medium']
+			} else if (sourceAudioDuration < 30 * 60) {
+				// If up to 30 minutes, set granularity to low, single pass
+				granularities = ['low']
+			} else {
+				// Otherwisek, use multipass processing, first with xx-low granularity, then low
+				granularities = ['xx-low', 'low']
+			}
 		}
 
-		if (typeof options.dtw!.windowDuration == 'number') {
-			if (granularities.length == 1) {
+		if (options.dtw!.windowDuration) {
+			if (typeof options.dtw!.windowDuration === 'number') {
 				windowDurations = [options.dtw!.windowDuration]
-			} else if (granularities.length == 2) {
-				windowDurations = [options.dtw!.windowDuration, 15]
+			} else if (Array.isArray(options.dtw!.windowDuration)) {
+				windowDurations = options.dtw!.windowDuration
 			} else {
-				throw new Error(`More than two passes requested, this requires window durations to be explicitly specified for each pass. For example 'dtw.windowDuration=[600,60,10]'.`)
+				throw new Error(`'dtw.windowDuration' must be a number or an array of numbers.`)
 			}
-		} else if (Array.isArray(options.dtw!.windowDuration)) {
-			windowDurations = options.dtw!.windowDuration
 		} else {
-			throw new Error('No window duration given')
+			if (granularities.length > 2) {
+				throw new Error(`More than two passes requested, this requires window durations to be explicitly specified for each pass. For example 'dtw.windowDuration=[600,60,10]'.`)
+			}
+
+			if (sourceAudioDuration < 5 * 60) {
+				// If up to 5 minutes, set window duration to one minute
+				windowDurations = [60]
+			} else if (sourceAudioDuration < 2.5 * 60 * 60) {
+				// If less than 2.5 hours, set window duration to 20% of total duration
+				windowDurations = [Math.ceil(sourceAudioDuration * 0.2)]
+			} else {
+				// Otherwise, set window duration to 30 minutes
+				windowDurations = [30 * 60]
+			}
+		}
+
+		if (granularities.length === 2 && windowDurations.length === 1) {
+			windowDurations = [windowDurations[0], 15]
 		}
 
 		if (granularities.length != windowDurations.length) {
-			throw new Error(`Unequal element counts in options. 'dtw.granularity' has ${granularities.length} items, but 'dtw.windowDuration' has ${windowDurations.length} items. Can't infer what number of DTW passes were intended.`)
+			throw new Error(`The option 'dtw.granularity' has ${granularities.length} values, but 'dtw.windowDuration' has ${windowDurations.length} values. The lengths should be equal.`)
 		}
 
 		return { windowDurations, granularities }
@@ -140,6 +157,8 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 	switch (options.engine) {
 		case 'dtw': {
+			const { windowDurations, granularities } = getDtwWindowGranularitiesAndDurations()
+
 			logger.end()
 
 			const {
@@ -149,14 +168,14 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 			logger.end()
 
-			const { windowDurations, granularities } = getDtwWindowDurationsAndGranularities()
-
 			mappedTimeline = await alignUsingDtw(sourceRawAudio, referenceRawAudio, referenceTimeline, granularities, windowDurations)
 
 			break
 		}
 
 		case 'dtw-ra': {
+			const { windowDurations, granularities } = getDtwWindowGranularitiesAndDurations()
+
 			logger.end()
 
 			const recognitionOptions: API.RecognitionOptions =
@@ -181,8 +200,6 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 			logger.end()
 
-			const { windowDurations, granularities } = getDtwWindowDurationsAndGranularities()
-
 			const phoneAlignmentMethod = options.dtw!.phoneAlignmentMethod!
 
 			const espeakOptions: EspeakOptions = {
@@ -229,6 +246,8 @@ export async function align(input: AudioSourceParam, transcript: string, options
 		}
 	}
 
+	logger.start(`Postprocess timeline`)
+
 	// If the audio was cropped before recognition, map the timestamps back to the original audio
 	if (sourceUncropTimeline && sourceUncropTimeline.length > 0) {
 		API.convertCroppedToUncroppedTimeline(mappedTimeline, sourceUncropTimeline)
@@ -354,7 +373,7 @@ export const defaultAlignmentOptions: AlignmentOptions = {
 	},
 
 	dtw: {
-		granularity: 'auto',
+		granularity: undefined,
 		windowDuration: undefined,
 		phoneAlignmentMethod: 'dtw'
 	},

package/src/api/LanguageDetection.ts

@@ -257,7 +257,7 @@ export async function detectTextLanguage(input: string, options: TextLanguageDet
 		case 'tinyld': {
 			const { detectLanguage } = await import('../text-language-detection/TinyLDLanguageDetection.js')
 
-			logger.start('Detecting text language using tinyld')
+			logger.start('Detect text language using tinyld')
 
 			detectedLanguageProbabilities = await detectLanguage(input)
 
@@ -267,7 +267,7 @@ export async function detectTextLanguage(input: string, options: TextLanguageDet
 		case 'fasttext': {
 			const { detectLanguage } = await import('../text-language-detection/FastTextLanguageDetection.js')
 
-			logger.start('Detecting text language using FastText')
+			logger.start('Detect text language using FastText')
 
 			detectedLanguageProbabilities = await detectLanguage(input)
 

package/src/api/Synthesis.ts

@@ -54,7 +54,7 @@ async function synthesizeSegments(segments: string[], options: SynthesisOptions,
 	options = extendDeep(defaultSynthesisOptions, options)
 
 	if (!options.language && !options.voice) {
-		logger.start('No language or voice specified. Detecting language')
+		logger.start('No language or voice specified. Detect language')
 
 		let segmentsPlainText = segments
 

package/src/synthesis/EspeakTTS.ts

@@ -44,7 +44,7 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 
 	for (let i = 0; i < words.length; i++) {
 		const currentWord = words[i]
-		const previousWord = words[i-1]
+		const previousWord = words[i - 1]
 
 		if (i > 0 && currentWord == previousWord && !wordCharacterPattern.test(currentWord)) {
 			wordsWithMerges[wordsWithMerges.length - 1] += currentWord
@@ -62,6 +62,12 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 
 	const simplifiedFragments = normalizedFragments.map(word => simplifyPunctuationCharacters(word).toLocaleLowerCase())
 
+	for (let i = 0; i < normalizedFragments.length; i++) {
+		if ([`'`, `"`].includes(simplifiedFragments[i])) {
+			normalizedFragments[i] = '()'
+		}
+	}
+
 	for (let fragmentIndex = 0; fragmentIndex < normalizedFragments.length; fragmentIndex++) {
 		const fragment = normalizedFragments[fragmentIndex]
 

package/src/utilities/Compression.ts

@@ -110,3 +110,18 @@ export async function getDeflateCompressionMetricsForString(str: string) {
 		ratio: originalStringBytes.length / compressedStringBytes.length
 	}
 }
+
+export function computeDeltas(data: Float32Array) {
+	const deltas = new Float32Array(data.length)
+
+	let val = 0
+
+	for (let i = 0; i < data.length; i++) {
+		const delta = Math.floor(data[i] - val)
+		deltas[i] = delta
+
+		val += delta
+	}
+
+	return deltas
+}