echogarden 1.8.2 → 1.8.4

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "1.8.2",
+	"version": "1.8.4",
 	"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -85,7 +85,7 @@
 		"jsdom": "^25.0.1",
 		"json5": "^2.2.3",
 		"kuromoji": "^0.1.2",
-		"microsoft-cognitiveservices-speech-sdk": "^1.40.0",
+		"microsoft-cognitiveservices-speech-sdk": "^1.41.0",
 		"moving-median": "^1.0.0",
 		"msgpack-lite": "^0.1.26",
 		"onnxruntime-node": "^1.19.2",

package/src/alignment/DTWMfccSequenceAlignment.ts

@@ -1,10 +1,10 @@
-import { cosineDistancePrecomputedMagnitudes, createVectorForIntegerRange, euclideanDistance, euclideanDistance13Dim, magnitude } from '../math/VectorMath.js'
-import { logToStderr } from '../utilities/Utilities.js'
+import { cosineDistancePrecomputedMagnitudes, euclideanDistance, euclideanDistance13Dim, magnitude } from '../math/VectorMath.js'
+import { getIntegerRange, logToStderr } from '../utilities/Utilities.js'
 import { alignDTWWindowed } from './DTWSequenceAlignmentWindowed.js'
 
 const log = logToStderr
 
-export async function alignMFCC_DTW(mfccFrames1: number[][], mfccFrames2: number[][], windowLength: number, distanceFunctionKind: 'euclidean' | 'cosine' = 'euclidean', centerIndexes?: number[]) {
+export async function alignMFCC_DTW(mfccFrames1: ArrayLike<number>[], mfccFrames2: ArrayLike<number>[], windowLength: number, distanceFunctionKind: 'euclidean' | 'cosine' = 'euclidean', centerIndexes?: number[]) {
 	if (distanceFunctionKind == 'euclidean') {
 		let distanceFunction = euclideanDistance
 
@@ -22,16 +22,19 @@ export async function alignMFCC_DTW(mfccFrames1: number[][], mfccFrames2: number
 
 		return path
 	} else if (distanceFunctionKind == 'cosine') {
-		const indexes1 = createVectorForIntegerRange(0, mfccFrames1.length)
-		const indexes2 = createVectorForIntegerRange(0, mfccFrames2.length)
+		const indexes1 = getIntegerRange(0, mfccFrames1.length)
+		const indexes2 = getIntegerRange(0, mfccFrames2.length)
 
 		const magnitudes1 = mfccFrames1.map(magnitude)
 		const magnitudes2 = mfccFrames2.map(magnitude)
 
+		const distanceFunction = (i: number, j: number) =>
+			cosineDistancePrecomputedMagnitudes(mfccFrames1[i], mfccFrames2[j], magnitudes1[i], magnitudes2[j])
+
 		const { path } = alignDTWWindowed(
 			indexes1,
 			indexes2,
-			(i, j) => cosineDistancePrecomputedMagnitudes(mfccFrames1[i], mfccFrames2[j], magnitudes1[i], magnitudes2[j]),
+			distanceFunction,
 			windowLength,
 			centerIndexes
 		)

package/src/alignment/DTWSequenceAlignment.ts

@@ -3,7 +3,7 @@ import { AlignmentPath } from './SpeechAlignment.js'
 
 const log = logToStderr
 
-export function alignDTW<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, deletionEnabled = true) {
+export function alignDTW<T, U>(sequence1: ArrayLike<T>, sequence2: ArrayLike<U>, costFunction: (a: T, b: U) => number, deletionEnabled = true) {
 	if (sequence1.length == 0 || sequence2.length == 0) {
 		return { path: [] as AlignmentPath, pathCost: 0 }
 	}
@@ -23,7 +23,7 @@ export function alignDTW<T, U>(sequence1: T[], sequence2: U[], costFunction: (a:
 	return { path, pathCost }
 }
 
-function computeAccumulatedCostMatrix<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, deletionEnabled = true) {
+function computeAccumulatedCostMatrix<T, U>(sequence1: ArrayLike<T>, sequence2: ArrayLike<U>, costFunction: (a: T, b: U) => number, deletionEnabled = true) {
 	const rowCount = sequence2.length
 	const columnCount = sequence1.length
 

package/src/alignment/DTWSequenceAlignmentWindowed.ts

@@ -3,7 +3,7 @@ import { AlignmentPath } from './SpeechAlignment.js'
 
 const log = logToStderr
 
-export function alignDTWWindowed<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, windowMaxLength: number, centerIndexes?: number[]) {
+export function alignDTWWindowed<T, U>(sequence1: ArrayLike<T>, sequence2: ArrayLike<U>, costFunction: (a: T, b: U) => number, windowMaxLength: number, centerIndexes?: ArrayLike<number>) {
 	windowMaxLength = Math.max(windowMaxLength, 2)
 
 	if (sequence1.length == 0 || sequence2.length == 0) {
@@ -29,7 +29,7 @@ export function alignDTWWindowed<T, U>(sequence1: T[], sequence2: U[], costFunct
 	return { path, pathCost }
 }
 
-function computeAccumulatedCostMatrixTransposed<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, windowMaxLength: number, centerIndexes?: number[]) {
+function computeAccumulatedCostMatrixTransposed<T, U>(sequence1: ArrayLike<T>, sequence2: ArrayLike<U>, costFunction: (a: T, b: U) => number, windowMaxLength: number, centerIndexes?: ArrayLike<number>) {
 	const halfWindowMaxLength = Math.floor(windowMaxLength / 2)
 
 	const columnCount = sequence1.length
@@ -139,7 +139,7 @@ function computeAccumulatedCostMatrixTransposed<T, U>(sequence1: T[], sequence2:
 	}
 }
 
-function computeBestPathTransposed(accumulatedCostMatrixTransposed: Float32Array[], windowStartOffsets: Int32Array) {
+function computeBestPathTransposed(accumulatedCostMatrixTransposed: ArrayLike<number>[], windowStartOffsets: ArrayLike<number>) {
 	const columnCount = accumulatedCostMatrixTransposed.length
 	const rowCount = accumulatedCostMatrixTransposed[0].length
 

package/src/alignment/SpeechAlignment.ts

@@ -716,7 +716,7 @@ function getMappedFrameIndexForPath(referenceFrameIndex: number, compactedPath:
 	return mappedFrameIndex
 }
 
-function getMfccOptionsForGranularity(granularity: DtwGranularity) {
+export function getMfccOptionsForGranularity(granularity: DtwGranularity) {
 	let mfccOptions: MfccOptions
 
 	if (granularity == 'xx-low') {

package/src/api/SpeechSearch.ts

@@ -0,0 +1,12 @@
+import { AudioSourceParam } from "../audio/AudioUtilities.js";
+
+export async function searchSpeech(inputAudio: AudioSourceParam, text: string, options: SpeechSearchOptions): Promise<SpeechSearchResult> {
+	return {}
+}
+
+export interface SpeechSearchOptions {
+
+}
+
+export interface SpeechSearchResult {
+}

package/src/dsp/MFCC.ts

@@ -36,8 +36,6 @@ export async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions = {
 	logger.start(`Resample audio to analysis sample rate (${analysisSampleRate}Hz)`)
 	const resampledAudio = await resampleAudioSpeex(monoAudio, analysisSampleRate)
 
-	let mfccs: number[][]
-
 	if (emphasisFactor > 0) {
 		logger.start('Apply emphasis')
 		resampledAudio.audioChannels[0] = applyEmphasis(resampledAudio.audioChannels[0], emphasisFactor)
@@ -47,20 +45,19 @@ export async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions = {
 	const { melSpectogram } = await computeMelSpectogram(resampledAudio, fftOrder, windowSize, hopLength, filterbankCount, lowerFrequencyHz, upperFrequencyHz)
 
 	logger.start('Extract MFCCs from Mel spectogram')
-	const mfccsFloat32 = melSpectogramToMFCCs(melSpectogram, featureCount)
-
-	mfccs = mfccsFloat32.map(mfcc => Array.from(mfcc))
+	let mfccs = melSpectogramToMFCCs(melSpectogram, featureCount)
 
 	if (options.normalize!) {
 		logger.start('Normalize MFCCs')
 
 		const { normalizedVectors, mean, stdDeviation } = normalizeVectors(mfccs)
+
 		mfccs = normalizedVectors
-		//mfccs = mfccs.map(mfcc => subtractVectors(mfcc, mean))
 	}
 
 	if (lifteringFactor > 0) {
 		logger.start('Apply liftering to MFCCs')
+
 		mfccs = applyLiftering(mfccs, lifteringFactor)
 	}
 
@@ -75,7 +72,7 @@ export async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions = {
 	return mfccs
 }
 
-export function melSpectogramToMFCCs(melSpectogram: Float32Array[], mfccFeatureCount: number) {
+export function melSpectogramToMFCCs(melSpectogram: ArrayLike<number>[], mfccFeatureCount: number) {
 	const melBandCount = melSpectogram[0].length
 	const dctMatrix = createDCTType2CoefficientMatrix(mfccFeatureCount, melBandCount)
 
@@ -84,7 +81,7 @@ export function melSpectogramToMFCCs(melSpectogram: Float32Array[], mfccFeatureC
 	return mfccs
 }
 
-export function melSpectrumToMFCC(melSpectrum: Float32Array, mfccFeatureCount: number, dctMatrix: Float32Array[], normalization: 'none' | 'orthonormal' = 'orthonormal') {
+export function melSpectrumToMFCC(melSpectrum: ArrayLike<number>, mfccFeatureCount: number, dctMatrix: ArrayLike<number>[], normalization: 'none' | 'orthonormal' = 'orthonormal') {
 	const melBandCount = melSpectrum.length
 
 	let firstFeatureNormalizationFactor: number
@@ -108,7 +105,6 @@ export function melSpectrumToMFCC(melSpectrum: Float32Array, mfccFeatureCount: n
 		for (let j = 0; j < melBandCount; j++) {
 			const dctCoefficient = dctMatrixRow[j]
 			const logMel = powerToDecibels(melSpectrum[j])
-			//const logMel = Math.log(1e-40 + melSpectrum[j])
 
 			sum += dctCoefficient * logMel
 		}
@@ -123,7 +119,7 @@ export function melSpectrumToMFCC(melSpectrum: Float32Array, mfccFeatureCount: n
 }
 
 export function createDCTType2CoefficientMatrix(mfccFeatureCount: number, melBandCount: number) {
-	const dctMatrix = new Array<Float32Array>(mfccFeatureCount)
+	const dctMatrix: Float32Array[] = []
 
 	for (let mfccFeatureIndex = 0; mfccFeatureIndex < mfccFeatureCount; mfccFeatureIndex++) {
 		const row = new Float32Array(melBandCount)
@@ -134,29 +130,13 @@ export function createDCTType2CoefficientMatrix(mfccFeatureCount: number, melBan
 			row[melBandIndex] = Math.cos(innerMultiplier * (melBandIndex + 0.5))
 		}
 
-		dctMatrix[mfccFeatureIndex] = row
+		dctMatrix.push(row)
 	}
 
 	return dctMatrix
 }
 
-export function mfccBufferToVectors(mfccBuffer: Float64Array, mfccFeatureCount: number) {
-	if (mfccBuffer.length % mfccFeatureCount != 0) {
-		throw new Error(`MFCC buffer length is not a multiple of the expected feature count (${mfccFeatureCount})`)
-	}
-
-	const mfccVectors: number[][] = []
-
-	for (let offset = 0; offset < mfccBuffer.length; offset += mfccFeatureCount) {
-		const mfccVector = Array.from(mfccBuffer.subarray(offset, offset + mfccFeatureCount))
-
-		mfccVectors.push(mfccVector)
-	}
-
-	return mfccVectors
-}
-
-export function applyEmphasis(samples: Float32Array, emphasisFactor = 0.97, initialState = 0) {
+export function applyEmphasis(samples: ArrayLike<number>, emphasisFactor = 0.97, initialState = 0) {
 	const processedSamples = new Float32Array(samples.length)
 
 	processedSamples[0] = samples[0] - (emphasisFactor * initialState)
@@ -168,7 +148,7 @@ export function applyEmphasis(samples: Float32Array, emphasisFactor = 0.97, init
 	return processedSamples
 }
 
-export function applyLiftering(mfccs: number[][], lifteringFactor: number) {
+export function applyLiftering(mfccs: ArrayLike<number>[], lifteringFactor: number) {
 	const featureCount = mfccs[0].length
 
 	const lifterMultipliers = new Float32Array(featureCount)
@@ -177,10 +157,10 @@ export function applyLiftering(mfccs: number[][], lifteringFactor: number) {
 		lifterMultipliers[i] = 1 + (lifteringFactor / 2) * Math.sin(Math.PI * (i + 1) / lifteringFactor)
 	}
 
-	const lifteredMfccs: number[][] = []
+	const lifteredMfccs: Float32Array[] = []
 
 	for (const mfcc of mfccs) {
-		const lifteredMfcc = new Array(featureCount)
+		const lifteredMfcc = new Float32Array(featureCount)
 
 		for (let i = 0; i < featureCount; i++) {
 			lifteredMfcc[i] = mfcc[i] * lifterMultipliers[i]

package/src/math/MedianFilter.ts

@@ -1,6 +1,6 @@
 import { createVector } from "./VectorMath.js"
 
-export function medianOf5Filter(points: number[]) {
+export function medianOf5Filter(points: ArrayLike<number>): Float32Array {
 	// This function computes the moving median with a window of 5 elements.
 
 	// I initialized the window such that at the edges of the range no median would be computed.
@@ -10,10 +10,10 @@ export function medianOf5Filter(points: number[]) {
 	const pointCount = points.length
 
 	if (pointCount < 5) {
-		return points
+		return Float32Array.from(points)
 	}
 
-	const medians = createVector(pointCount)
+	const medians = new Float32Array(pointCount)
 
 	medians[0] = points[0]
 	medians[1] = points[1]
@@ -38,7 +38,7 @@ export function medianOf3Filter(points: ArrayLike<number>) {
 		return points
 	}
 
-	const medians: number[] = createVector(pointCount)
+	const medians = new Float32Array(pointCount)
 
 	medians[0] = points[0]
 	medians[pointCount - 1] = points[pointCount - 1]
@@ -110,7 +110,7 @@ export function medianOf3(a: number, b: number, c: number) {
 }
 
 // Slower, variable-width median filter using the `moving-median` package
-export async function medianFilter(points: number[], width: number) {
+export async function medianFilter(points: ArrayLike<number>, width: number) {
 	const { default: createMedianFilter } = await import('moving-median')
 
 	const filter = createMedianFilter(width)