echogarden 0.11.12 → 0.11.13

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "0.11.12",
+	"version": "0.11.13",
 	"description": "An integrated speech system, providing a variety of synthesis, recognition, alignment, and other processing tools, designed to be directly accessible to end-users.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -37,6 +37,7 @@
 	"main": "./dist/API/API.js",
 	"type": "module",
 	"files": [
+		"src",
 		"dist",
 		"data",
 		"docs",
@@ -53,8 +54,8 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "^3.409.0",
-		"@aws-sdk/client-transcribe-streaming": "^3.409.0",
+		"@aws-sdk/client-polly": "^3.410.0",
+		"@aws-sdk/client-transcribe-streaming": "^3.410.0",
 		"@echogarden/espeak-ng-emscripten": "^0.1.2",
 		"@echogarden/fasttext-wasm": "^0.1.0",
 		"@echogarden/flite-wasi": "^0.1.1",
@@ -121,10 +122,10 @@
 		"@types/ndarray-ops": "^1.2.4",
 		"@types/node": "^20.6.0",
 		"@types/recursive-readdir": "^2.2.1",
-		"@types/tar": "^6.1.5",
+		"@types/tar": "^6.1.6",
 		"@types/ws": "^8.5.5",
-		"@typescript-eslint/eslint-plugin": "^6.6.0",
-		"@typescript-eslint/parser": "^6.6.0",
+		"@typescript-eslint/eslint-plugin": "^6.7.0",
+		"@typescript-eslint/parser": "^6.7.0",
 		"@types/graceful-fs": "^4.1.6",
 		"eslint": "^8.49.0",
 		"ts-json-schema-generator": "^1.3.0",

package/src/alignment/DTWMfccSequenceAlignment.ts

@@ -0,0 +1,43 @@
+import { cosineDistancePrecomputedMagnitudes, createVectorForIntegerRange, eucledianDistance, magnitude } from "../math/VectorMath.js"
+import { logToStderr } from "../utilities/Utilities.js"
+import { alignDTWWindowed } from "./DTWSequenceAlignmentWindowed.js"
+
+const log = logToStderr
+
+export async function alignMFCC_DTW(mfccFrames1: number[][], mfccFrames2: number[][], windowLength: number, distanceFunction: "eucledian" | "cosine" = "eucledian", centerIndexes?: number[]) {
+	if (distanceFunction == "eucledian") {
+		const { path } = alignDTWWindowed(
+			mfccFrames1,
+			mfccFrames2,
+			eucledianDistance,
+			windowLength,
+			centerIndexes
+		)
+
+		return path
+	} else if (distanceFunction == "cosine") {
+		const indexes1 = createVectorForIntegerRange(0, mfccFrames1.length)
+		const indexes2 = createVectorForIntegerRange(0, mfccFrames2.length)
+
+		const magnitudes1 = mfccFrames1.map(magnitude)
+		const magnitudes2 = mfccFrames2.map(magnitude)
+
+		const { path } = alignDTWWindowed(
+			indexes1,
+			indexes2,
+			(i, j) => cosineDistancePrecomputedMagnitudes(mfccFrames1[i], mfccFrames2[j], magnitudes1[i], magnitudes2[j]),
+			windowLength,
+			centerIndexes
+		)
+
+		return path
+	} else {
+		throw new Error("Invalid distance function")
+	}
+}
+
+export function getCostMatrixMemorySizeMB(sequence1Length: number, sequence2Length: number, windowLength: number) {
+	const costMatrixMemorySizeMB = sequence1Length * Math.min(sequence2Length, windowLength) * 4 / 1000000
+	
+	return costMatrixMemorySizeMB
+}

package/src/alignment/DTWSequenceAlignment.ts

@@ -0,0 +1,121 @@
+import { logToStderr } from "../utilities/Utilities.js"
+import { AlignmentPath } from "./SpeechAlignment.js"
+
+const log = logToStderr
+
+export function alignDTW<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, deletionEnabled = true) {
+	if (sequence1.length == 0 || sequence2.length == 0) {
+		return { path: [] as AlignmentPath, pathCost: 0 }
+	}
+
+	const rowCount = sequence2.length
+	const columnCount = sequence1.length
+
+	// Compute accumulated cost matrix
+	const accumulatedCostMatrix = computeAccumulatedCostMatrix(sequence1, sequence2, costFunction, deletionEnabled)
+
+	// Find best path for the computed matrix
+	const path = computeBestPath(accumulatedCostMatrix, deletionEnabled)
+
+	// Best path cost is the bottom right element of the matrix
+	const pathCost = accumulatedCostMatrix[rowCount - 1][columnCount - 1]
+
+	return { path, pathCost }
+}
+
+function computeAccumulatedCostMatrix<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, deletionEnabled = true) {
+	const rowCount = sequence2.length
+	const columnCount = sequence1.length
+
+	const accumulatedCostMatrix: Float32Array[] = new Array<Float32Array>(rowCount)
+
+	// Initialize matrix
+	for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+		accumulatedCostMatrix[rowIndex] = new Float32Array(columnCount)
+	}
+
+	for (let i = 1; i < columnCount; i++) {
+		accumulatedCostMatrix[0][i] = Infinity
+	}
+
+	for (let i = 1; i < rowCount; i++) {
+		accumulatedCostMatrix[i][0] = Infinity
+	}
+
+	// Fill out the matrix, go row by row
+	for (let rowIndex = 1; rowIndex < rowCount; rowIndex++) {
+		const previousRow = accumulatedCostMatrix[rowIndex - 1]
+		const currentRow = accumulatedCostMatrix[rowIndex]
+
+		for (let columnIndex = 1; columnIndex < columnCount; columnIndex++) {
+			const cost = costFunction(sequence1[columnIndex], sequence2[rowIndex])
+
+			const up = previousRow[columnIndex] // insertion
+			const left = currentRow[columnIndex - 1] // deletion
+			const upAndLeft = previousRow[columnIndex - 1] // match
+
+			currentRow[columnIndex] = cost + minimumOf3(up, deletionEnabled ? left : Infinity, upAndLeft)
+		}
+	}
+
+	return accumulatedCostMatrix
+}
+
+function computeBestPath(costMatrix: Float32Array[], deletionEnabled = true) {
+	const rowCount = costMatrix.length
+	const columnCount = costMatrix[0].length
+
+	const bestPath: AlignmentPath = []
+
+	let rowIndex = rowCount - 1
+	let columnIndex = columnCount - 1
+
+	bestPath.push({
+		source: columnIndex,
+		dest: rowIndex
+	})
+
+	while (rowIndex > 0 || columnIndex > 0) {
+		const up = costMatrix[rowIndex - 1][columnIndex]
+		const left = costMatrix[rowIndex][columnIndex - 1]
+		const upAndLeft = costMatrix[rowIndex - 1][columnIndex - 1]
+
+		const smallestCostDirection = argIndexOfMinimumOf3(up, deletionEnabled ? left : Infinity, upAndLeft)
+
+		if (smallestCostDirection == 1) {
+			rowIndex -= 1
+		} else if (smallestCostDirection == 2) {
+			columnIndex -= 1
+		} else {
+			rowIndex -= 1
+			columnIndex -= 1
+		}
+
+		bestPath.push({
+			source: columnIndex,
+			dest: rowIndex
+		})
+	}
+
+	return bestPath.reverse() as AlignmentPath
+}
+
+function minimumOf3(x1: number, x2: number, x3: number) {
+	if (x1 <= x2 && x1 <= x3) {
+		return x1
+	} else if (x2 <= x3) {
+		return x2
+	} else {
+		return x3
+	}
+}
+
+function argIndexOfMinimumOf3(x1: number, x2: number, x3: number) {
+	if (x1 <= x2 && x1 <= x3) {
+		return 1
+	} else if (x2 <= x3) {
+		return 2
+	} else {
+		return 3
+	}
+}

package/src/alignment/DTWSequenceAlignmentWindowed.ts

@@ -0,0 +1,210 @@
+import { logToStderr } from "../utilities/Utilities.js"
+import { AlignmentPath } from "./SpeechAlignment.js"
+
+const log = logToStderr
+
+export function alignDTWWindowed<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, windowMaxLength: number, centerIndexes?: number[]) {
+	if (windowMaxLength < 2) {
+		throw new Error("Window length must be greater or equal to 2")
+	}
+
+	if (sequence1.length == 0 || sequence2.length == 0) {
+		return { path: [] as AlignmentPath, pathCost: 0 }
+	}
+
+	// Compute accumulated cost matrix (transposed)
+	const { accumulatedCostMatrixTransposed, windowStartOffsets } = computeAccumulatedCostMatrixTransposed(sequence1, sequence2, costFunction, windowMaxLength, centerIndexes)
+
+	// Find best path for the computed matrix
+	const path = computeBestPathTransposed(accumulatedCostMatrixTransposed, windowStartOffsets)
+
+	// Best path cost is the bottom right element of the matrix
+	const columnCount = accumulatedCostMatrixTransposed.length
+	const rowCount = accumulatedCostMatrixTransposed[0].length
+
+	const pathCost = accumulatedCostMatrixTransposed[columnCount - 1][rowCount - 1]
+
+	// Return
+	return { path, pathCost }
+}
+
+function computeAccumulatedCostMatrixTransposed<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, windowMaxLength: number, centerIndexes?: number[]) {
+	const halfWindowMaxLength = Math.floor(windowMaxLength / 2)
+
+	const columnCount = sequence1.length
+	const rowCount = Math.min(windowMaxLength, sequence2.length)
+
+	const accumulatedCostMatrixTransposed: Float32Array[] = new Array<Float32Array>(columnCount)
+
+	// Initialize window start offsets array
+	const windowStartOffsets = new Int32Array(columnCount)
+
+	// Compute matrix column by column
+	for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) {
+		// Create new column and add it to the matrix
+		const currentColumn = new Float32Array(rowCount)
+		accumulatedCostMatrixTransposed[columnIndex] = currentColumn
+
+		// Compute window center, or use given one
+		let windowCenter: number
+
+		if (centerIndexes) {
+			windowCenter = centerIndexes[columnIndex]
+		} else {
+			windowCenter = Math.floor((columnIndex / columnCount) * sequence2.length)
+		}
+
+		// Compute window start and end offsets
+		let windowStartOffset = Math.max(windowCenter - halfWindowMaxLength, 0)
+		let windowEndOffset = windowStartOffset + rowCount
+
+		if (windowEndOffset > sequence2.length) {
+			windowEndOffset = sequence2.length
+			windowStartOffset = windowEndOffset - rowCount
+		}
+
+		// Store the start offset in the array
+		windowStartOffsets[columnIndex] = windowStartOffset
+
+		// Get target sequence1 value
+		const targetSequence1Value = sequence1[columnIndex]
+
+		// If first column, fill it only using the 'up' neighbor
+		if (columnIndex == 0) {
+			for (let rowIndex = 1; rowIndex < rowCount; rowIndex++) {
+				const cost = costFunction(targetSequence1Value, sequence2[windowStartOffset + rowIndex])
+				const upCost = currentColumn[rowIndex - 1]
+
+				currentColumn[rowIndex] = cost + upCost
+			}
+
+			continue
+		}
+
+		// If not first column
+		const leftColumn = accumulatedCostMatrixTransposed[columnIndex - 1]
+
+		// Compute the delta between the current window offset and previous column's window offset
+		const windowOffsetDelta = windowStartOffset - windowStartOffsets[columnIndex - 1]
+
+		for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+			// Compute the cost for current cell
+			const cost = costFunction(targetSequence1Value, sequence2[windowStartOffset + rowIndex])
+
+			// Retrieve the cost for the 'up' (insertion) neighbor
+			let upCost = Infinity
+			if (rowIndex > 0) {
+				upCost = currentColumn[rowIndex - 1]
+			}
+
+			// Retrieve the cost for the 'left' (deletion) neighbor
+			let leftCost = Infinity
+			const leftRowIndex = rowIndex + windowOffsetDelta
+
+			if (leftRowIndex < rowCount) {
+				leftCost = leftColumn[leftRowIndex]
+			}
+
+			// Retrieve the cost for the 'up and left' (match) neighbor
+			let upAndLeftCost = Infinity
+			const upAndLeftRowIndex = leftRowIndex - 1
+
+			if (upAndLeftRowIndex >= 0 && upAndLeftRowIndex < rowCount) {
+				upAndLeftCost = leftColumn[upAndLeftRowIndex]
+			}
+
+			// Write cost + minimum neighbor cost to the current column
+			currentColumn[rowIndex] = cost + minimumOf3(upCost, leftCost, upAndLeftCost)
+		}
+	}
+
+	return { accumulatedCostMatrixTransposed, windowStartOffsets }
+}
+
+function computeBestPathTransposed(accumulatedCostMatrixTransposed: Float32Array[], windowStartOffsets: Int32Array) {
+	const columnCount = accumulatedCostMatrixTransposed.length
+	const rowCount = accumulatedCostMatrixTransposed[0].length
+
+	const bestPath: AlignmentPath = []
+
+	let columnIndex = columnCount - 1
+	let rowIndex = rowCount - 1
+
+	while (columnIndex > 0 || rowIndex > 0) {
+		const windowStartIndex = windowStartOffsets[columnIndex]
+		const windowStartDelta = columnIndex > 0 ? windowStartIndex - windowStartOffsets[columnIndex - 1] : 0
+
+		bestPath.push({
+			source: columnIndex,
+			dest: windowStartIndex + rowIndex
+		})
+
+		const upRowIndex = rowIndex - 1
+		const upColumnIndex = columnIndex
+		let upCost = Infinity
+
+		if (upRowIndex >= 0) {
+			upCost = accumulatedCostMatrixTransposed[upColumnIndex][upRowIndex] // insertion
+		}
+
+		const leftRowIndex = rowIndex + windowStartDelta
+		const leftColumnIndex = columnIndex - 1
+		let leftCost = Infinity
+
+		if (leftColumnIndex >= 0 && leftRowIndex < rowCount) {
+			leftCost = accumulatedCostMatrixTransposed[leftColumnIndex][leftRowIndex] // deletion
+		}
+
+		const upAndLeftRowIndex = rowIndex - 1 + windowStartDelta
+		const upAndLeftColumnIndex = columnIndex - 1
+		let upAndLeftCost = Infinity
+
+		if (upAndLeftColumnIndex >= 0 && upAndLeftRowIndex >= 0 && upAndLeftRowIndex < rowCount) {
+			upAndLeftCost = accumulatedCostMatrixTransposed[upAndLeftColumnIndex][upAndLeftRowIndex] // match
+		}
+
+		if (upCost == Infinity && leftCost == Infinity && upAndLeftCost == Infinity) {
+			log(`computeBestPath: Unexpected - all cost directions are equal to infinity (${columnIndex}, ${rowIndex}).`)
+		}
+
+		const smallestCostDirection = argIndexOfMinimumOf3(upCost, leftCost, upAndLeftCost)
+
+		if (smallestCostDirection == 1) {
+			rowIndex = upRowIndex
+			columnIndex = upColumnIndex
+		} else if (smallestCostDirection == 2) {
+			rowIndex = leftRowIndex
+			columnIndex = leftColumnIndex
+		} else {
+			rowIndex = upAndLeftRowIndex
+			columnIndex = upAndLeftColumnIndex
+		}
+	}
+
+	bestPath.push({
+		source: 0,
+		dest: 0
+	})
+
+	return bestPath.reverse() as AlignmentPath
+}
+
+function minimumOf3(x1: number, x2: number, x3: number) {
+	if (x1 <= x2 && x1 <= x3) {
+		return x1
+	} else if (x2 <= x3) {
+		return x2
+	} else {
+		return x3
+	}
+}
+
+function argIndexOfMinimumOf3(x1: number, x2: number, x3: number) {
+	if (x1 <= x2 && x1 <= x3) {
+		return 1
+	} else if (x2 <= x3) {
+		return 2
+	} else {
+		return 3
+	}
+}

package/src/alignment/LevenshteinSequenceAlignment.ts

@@ -0,0 +1,126 @@
+import { logToStderr } from "../utilities/Utilities.js"
+import { AlignmentPath } from "./SpeechAlignment.js"
+
+const log = logToStderr
+
+export function alignLevenshtein<T, U>(sequence1: T[], sequence2: U[], getSubstitutionCost: (a: T, b: U) => number) {
+	const rowCount = sequence2.length + 1
+	const columnCount = sequence1.length + 1
+
+	// Compute accumulated cost matrix
+	const accumulatedCostMatrix = computeAccumulatedCostMatrix(sequence1, sequence2, getSubstitutionCost)
+
+	// Find best path for the computed matrix
+	const path = computeBestPath(accumulatedCostMatrix)
+
+	// Best path cost is the bottom right element of the matrix
+	const pathCost = accumulatedCostMatrix[rowCount - 1][columnCount - 1]
+
+	return { path, pathCost }
+}
+
+function computeAccumulatedCostMatrix<T, U>(sequence1: T[], sequence2: U[], getSubstitutionCost: (a: T, b: U) => number) {
+	const rowCount = sequence2.length + 1
+	const columnCount = sequence1.length + 1
+
+	const accumulatedCostMatrix: Float32Array[] = new Array<Float32Array>(rowCount)
+
+	// Allocate rows and initialize first column
+	for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+		const newRow = new Float32Array(columnCount)
+
+		newRow[0] = rowIndex
+		accumulatedCostMatrix[rowIndex] = newRow
+	}
+
+	// Initialize first row
+	for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) {
+		accumulatedCostMatrix[0][columnIndex] = columnIndex
+	}
+
+	// Fill out the rest of the matrix, go row by row
+	for (let rowIndex = 1; rowIndex < rowCount; rowIndex++) {
+		const previousRow = accumulatedCostMatrix[rowIndex - 1]
+		const currentRow = accumulatedCostMatrix[rowIndex]
+
+		for (let columnIndex = 1; columnIndex < columnCount; columnIndex++) {
+			const upAndLeft = previousRow[columnIndex - 1]
+			const up = previousRow[columnIndex]
+			const left = currentRow[columnIndex - 1]
+
+			const smallestCostDirection = argIndexOfSmallestOf3(upAndLeft, up, left)
+
+			let computedCostForCurrentElement: number
+
+			if (smallestCostDirection == 1) {
+				const subtitutionCost = getSubstitutionCost(sequence1[columnIndex - 1], sequence2[rowIndex - 1])
+
+				computedCostForCurrentElement = upAndLeft + subtitutionCost
+			} else if (smallestCostDirection == 2) {
+				computedCostForCurrentElement = up + 1
+			} else {
+				computedCostForCurrentElement = left + 1
+			}
+
+			currentRow[columnIndex] = computedCostForCurrentElement
+		}
+	}
+
+	return accumulatedCostMatrix
+}
+
+function computeBestPath(costMatrix: Float32Array[]) {
+	const bestPath: AlignmentPath = []
+
+	let rowIndex = costMatrix.length - 1
+	let columnIndex = costMatrix[0].length - 1
+
+	while (rowIndex > 1 || columnIndex > 1) {
+		bestPath.push({
+			source: columnIndex - 1,
+			dest: rowIndex - 1
+		})
+
+		if (rowIndex == 1) {
+			columnIndex -= 1
+			continue
+		}
+
+		if (columnIndex == 1) {
+			rowIndex -= 1
+			continue
+		}
+
+		const upAndLeft = costMatrix[rowIndex - 1][columnIndex - 1]
+		const up = costMatrix[rowIndex - 1][columnIndex]
+		const left = costMatrix[rowIndex][columnIndex - 1]
+
+		const smallestCostDirection = argIndexOfSmallestOf3(upAndLeft, up, left)
+
+		if (smallestCostDirection == 1) {
+			rowIndex -= 1
+			columnIndex -= 1
+		} else if (smallestCostDirection == 2) {
+			rowIndex -= 1
+		} else {
+			columnIndex -= 1
+		}
+	}
+
+	bestPath.push({
+		source: 0,
+		dest: 0
+	})
+
+	return bestPath.reverse() as AlignmentPath
+}
+
+function argIndexOfSmallestOf3(x1: number, x2: number, x3: number) {
+	if (x1 <= x2 && x1 <= x3) {
+		return 1
+	} else if (x2 <= x3) {
+		return 2
+	} else {
+		return 3
+	}
+}