echogarden 2.8.7 → 2.9.0

package/package.json

@@ -1,21 +1,21 @@
 {
 	"name": "echogarden",
-	"version": "2.8.7",
+	"version": "2.9.0",
 	"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
 	"keywords": [
 		"speech",
 		"text-to-speech",
-		"speech synthesis",
+		"speech-synthesis",
 		"speech-to-text",
-		"speech recognition",
-		"speech alignment",
-		"forced alignment",
-		"speech translation",
-		"language identification",
-		"language detection",
-		"source separation"
+		"speech-recognition",
+		"speech-alignment",
+		"forced-alignment",
+		"speech-translation",
+		"language-identification",
+		"language-detection",
+		"source-separation"
 	],
 	"repository": {
 		"type": "git",
@@ -52,8 +52,8 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "~3.812.0",
-		"@aws-sdk/client-transcribe-streaming": "~3.812.0",
+		"@aws-sdk/client-polly": "~3.817.0",
+		"@aws-sdk/client-transcribe-streaming": "~3.817.0",
 		"@echogarden/audio-io": "~0.4.2",
 		"@echogarden/espeak-ng-emscripten": "~0.3.3",
 		"@echogarden/fasttext-wasm": "~0.1.0",
@@ -85,7 +85,7 @@
 		"microsoft-cognitiveservices-speech-sdk": "~1.44.0",
 		"msgpack-lite": "~0.1.26",
 		"onnxruntime-node": "~1.21.1",
-		"openai": "~4.102.0",
+		"openai": "~4.103.0",
 		"regexp-composer": "~0.3.0",
 		"sam-js": "~0.3.1",
 		"strip-ansi": "~7.1.0",
@@ -114,7 +114,7 @@
 		"@types/graceful-fs": "~4.1.9",
 		"@types/jsdom": "~21.1.7",
 		"@types/msgpack-lite": "~0.1.11",
-		"@types/node": "~22.15.21",
+		"@types/node": "~22.15.23",
 		"@types/tar": "~6.1.13",
 		"@types/ws": "~8.18.1",
 		"ts-json-schema-generator": "~2.4.0",

package/src/api/Recognition.ts

@@ -1,26 +1,28 @@
+import chalk from 'chalk'
+
+import * as API from './API.js'
+
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 
 import { logToStderr } from '../utilities/Utilities.js'
 import { AudioSourceParam, RawAudio, ensureRawAudio, normalizeAudioLevelInPlace, trimAudioEnd } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 
-import * as API from './API.js'
 import { Timeline, addWordTextOffsetsToTimelineInPlace, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
 import { formatLanguageCodeWithName, parseLangIdentifier } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
-import chalk from 'chalk'
 
-import { type WhisperOptions } from '../recognition/WhisperSTT.js'
+import { type WhisperPartCallback, type WhisperOptions } from '../recognition/WhisperSTT.js'
 import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 import { type OpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
 import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
 import { type SileroRecognitionOptions } from '../recognition/SileroSTT.js'
 import { type DeepgramSTTOptions } from '../recognition/DeepgramSTT.js'
-import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
+import { type OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 
 const log = logToStderr
 
-export async function recognize(input: AudioSourceParam, options: RecognitionOptions): Promise<RecognitionResult> {
+export async function recognize(input: AudioSourceParam, options: RecognitionOptions, onPart?: WhisperPartCallback): Promise<RecognitionResult> {
 	const logger = new Logger()
 
 	const startTimestamp = logger.getTimestamp()
@@ -110,7 +112,8 @@ export async function recognize(input: AudioSourceParam, options: RecognitionOpt
 				modelDir,
 				'transcribe',
 				shortLanguageCode,
-				whisperOptions
+				whisperOptions,
+				onPart,
 			))
 
 			break
@@ -318,7 +321,7 @@ export async function recognize(input: AudioSourceParam, options: RecognitionOpt
 	const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(timeline, transcript, languageCode, 'single', 'preserve')
 
 	logger.end()
-	logger.logDuration('Total recognition time', startTimestamp, chalk.magentaBright)
+	logger.logDuration('\nTotal recognition time', startTimestamp, chalk.magentaBright)
 
 	return {
 		transcript,

package/src/api/SpeechTranslation.ts

@@ -1,3 +1,7 @@
+import chalk from 'chalk'
+
+import * as API from './API.js'
+
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 
 import { logToStderr } from '../utilities/Utilities.js'
@@ -5,14 +9,12 @@ import { AudioSourceParam, RawAudio, ensureRawAudio, normalizeAudioLevelInPlace,
 import { Logger } from '../utilities/Logger.js'
 
 import { Timeline, addWordTextOffsetsToTimelineInPlace, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
-import { type WhisperOptions } from '../recognition/WhisperSTT.js'
+import { WhisperPartCallback, type WhisperOptions } from '../recognition/WhisperSTT.js'
 import { formatLanguageCodeWithName, getShortLanguageCode, normalizeIdentifierToLanguageCode, parseLangIdentifier } from '../utilities/Locale.js'
-import { EngineMetadata } from './Common.js'
+import { type EngineMetadata } from './Common.js'
 import { type SpeechLanguageDetectionOptions, detectSpeechLanguage } from './API.js'
-import chalk from 'chalk'
 import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 
-import * as API from './API.js'
 import { type OpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
 import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
 
@@ -21,7 +23,7 @@ const log = logToStderr
 /////////////////////////////////////////////////////////////////////////////////////////////
 // Speech translation
 /////////////////////////////////////////////////////////////////////////////////////////////
-export async function translateSpeech(input: AudioSourceParam, options: SpeechTranslationOptions): Promise<SpeechTranslationResult> {
+export async function translateSpeech(input: AudioSourceParam, options: SpeechTranslationOptions, onPart?: WhisperPartCallback): Promise<SpeechTranslationResult> {
 	const logger = new Logger()
 
 	const startTimestamp = logger.getTimestamp()
@@ -122,7 +124,15 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
 
 			logger.end();
 
-			({ transcript, timeline: wordTimeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, 'translate', sourceLanguage, whisperOptions))
+			({ transcript, timeline: wordTimeline } = await WhisperSTT.recognize(
+				sourceRawAudio,
+				modelName,
+				modelDir,
+				'translate',
+				sourceLanguage,
+				whisperOptions,
+				onPart,
+			))
 
 			break
 		}

package/src/denoising/NSNet2.ts

@@ -58,8 +58,8 @@ export class NSNet2 {
 		logger.start('Compute STFT frames')
 		const stftrFrames = await stftr(rawAudio.audioChannels[0], fftSize, fftSize, fftHopSize, 'hann')
 
-		logger.start('Compute log-power spectogram')
-		let logPowerSpectogram: Float32Array[] = []
+		logger.start('Compute log-power spectrogram')
+		let logPowerSpectrogram: Float32Array[] = []
 
 		{
 			for (const frame of stftrFrames) {
@@ -79,19 +79,19 @@ export class NSNet2 {
 					logPowerSpectrum[writeOffset++] = logPowerValue
 				}
 
-				logPowerSpectogram.push(logPowerSpectrum)
+				logPowerSpectrogram.push(logPowerSpectrum)
 			}
 		}
 
-		logger.start('Process log-power spectogram using ONNX model')
+		logger.start('Process log-power spectrogram using ONNX model')
 
-		const frameCount = logPowerSpectogram.length
+		const frameCount = logPowerSpectrogram.length
 		let flattenedOutputTensor: Float32Array
 
 		{
 			const Onnx = await import('onnxruntime-node')
 
-			const flattenedFeatures = concatFloat32Arrays(logPowerSpectogram)
+			const flattenedFeatures = concatFloat32Arrays(logPowerSpectrogram)
 
 			const inputTensor = new Onnx.Tensor('float32', flattenedFeatures, [1, frameCount, fftRealBinCount])
 			const inputs = { input: inputTensor }
@@ -126,8 +126,8 @@ export class NSNet2 {
 			}
 		}
 
-		// Allow logPowerSpectogram to be garbage collected
-		logPowerSpectogram = undefined as any
+		// Allow logPowerSpectrogram to be garbage collected
+		logPowerSpectrogram = undefined as any
 
 		logger.start('Reconstruct filtered signal using inverse STFT')
 		const filteredSignal = await stiftr(stftrFrames, fftSize, fftSize, fftHopSize, 'hann')

package/src/dsp/FFT.ts

@@ -6,7 +6,9 @@ import { createWasmHeapManager } from 'wasm-heap-manager'
 export async function stftr(samples: Float32Array, fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType) {
 	const frames: Float32Array[] = []
 
-	for await (const frame of stftrGenerator(samples, fftOrder, windowSize, hopSize, windowType)) {
+	const stftrGenerator = await createStftrGenerator(samples, fftOrder, windowSize, hopSize, windowType)
+
+	for (const frame of stftrGenerator) {
 		frames.push(frame)
 	}
 
@@ -14,7 +16,7 @@ export async function stftr(samples: Float32Array, fftOrder: number, windowSize:
 }
 
 // Incrementally generate short-term Fourier transform frames (real-valued)
-export async function* stftrGenerator(samples: Float32Array, fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType) {
+export async function createStftrGenerator(samples: Float32Array, fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType) {
 	if (fftOrder % 2 !== 0 || windowSize % 2 !== 0) {
 		throw new Error('FFT order and window size must be multiples of 2')
 	}
@@ -49,28 +51,32 @@ export async function* stftrGenerator(samples: Float32Array, fftOrder: number, w
 	const binsBufferRef = wasmHeap.allocFloat32Array(fftOrder * 2)
 	const workBufferRef = wasmHeap.allocFloat32Array(fftOrder * 2)
 
-	for (let offset = 0; offset < sampleCount; offset += hopSize) {
-		const windowSamples = samples.subarray(offset, offset + windowSize)
-		frameBufferRef.clear()
+	function* stftrGenerator() {
+		for (let offset = 0; offset < sampleCount; offset += hopSize) {
+			const windowSamples = samples.subarray(offset, offset + windowSize)
+			frameBufferRef.clear()
 
-		const frameBufferView = frameBufferRef.view
+			const frameBufferView = frameBufferRef.view
 
-		for (let i = 0; i < windowSamples.length; i++) {
-			frameBufferView[i] = windowSamples[i] * windowWeights[i]
-		}
+			for (let i = 0; i < windowSamples.length; i++) {
+				frameBufferView[i] = windowSamples[i] * windowWeights[i]
+			}
 
-		binsBufferRef.clear()
+			binsBufferRef.clear()
 
-		m._pffft_transform_ordered(statePtr, frameBufferRef.address, binsBufferRef.address, workBufferRef.address, 0)
+			m._pffft_transform_ordered(statePtr, frameBufferRef.address, binsBufferRef.address, workBufferRef.address, 0)
 
-		const bins = binsBufferRef.view.slice(0, fftOrder + 2)
+			const bins = binsBufferRef.view.slice(0, fftOrder + 2)
 
-		yield bins
-	}
+			yield bins
+		}
 
-	m._pffft_destroy_setup(statePtr)
+		m._pffft_destroy_setup(statePtr)
 
-	wasmHeap.freeAll()
+		wasmHeap.freeAll()
+	}
+
+	return stftrGenerator()
 }
 
 // Compute short-term inverse Fourier transform (real-valued)
@@ -168,7 +174,7 @@ export function getBinFrequencies(binCount: number, maxFrequency: number) {
 }
 
 // Convert an array of raw FFT frames to a power spectrum
-export function fftFramesToPowerSpectogram(fftFrames: Float32Array[]) {
+export function fftFramesToPowerSpectrogram(fftFrames: Float32Array[]) {
 	return fftFrames.map(fftFrame => fftFrameToPowerSpectrum(fftFrame))
 }
 

package/src/dsp/MFCC.ts

@@ -1,7 +1,7 @@
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { resampleAudioSpeex } from './SpeexResampler.js'
-import { computeMelSpectogram } from './MelSpectogram.js'
+import { computeMelSpectrogram } from './MelSpectrogram.js'
 import { RawAudio, powerToDecibels } from '../audio/AudioUtilities.js'
 import { normalizeVectors } from '../math/VectorMath.js'
 
@@ -41,11 +41,11 @@ export async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions = {
 		resampledAudio.audioChannels[0] = applyEmphasis(resampledAudio.audioChannels[0], emphasisFactor)
 	}
 
-	logger.start('Compute Mel spectogram')
-	const { melSpectogram } = await computeMelSpectogram(resampledAudio, fftOrder, windowSize, hopLength, filterbankCount, lowerFrequencyHz, upperFrequencyHz)
+	logger.start('Compute Mel spectrogram')
+	const { melSpectrogram } = await computeMelSpectrogram(resampledAudio, fftOrder, windowSize, hopLength, filterbankCount, lowerFrequencyHz, upperFrequencyHz)
 
-	logger.start('Extract MFCCs from Mel spectogram')
-	let mfccs = melSpectogramToMFCCs(melSpectogram, featureCount)
+	logger.start('Extract MFCCs from Mel spectrogram')
+	let mfccs = melSpectrogramToMFCCs(melSpectrogram, featureCount)
 
 	if (options.normalize!) {
 		logger.start('Normalize MFCCs')
@@ -72,11 +72,11 @@ export async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions = {
 	return mfccs
 }
 
-export function melSpectogramToMFCCs(melSpectogram: ArrayLike<number>[], mfccFeatureCount: number) {
-	const melBandCount = melSpectogram[0].length
+export function melSpectrogramToMFCCs(melSpectrogram: ArrayLike<number>[], mfccFeatureCount: number) {
+	const melBandCount = melSpectrogram[0].length
 	const dctMatrix = createDCTType2CoefficientMatrix(mfccFeatureCount, melBandCount)
 
-	const mfccs = melSpectogram.map(frame => melSpectrumToMFCC(frame, mfccFeatureCount, dctMatrix))
+	const mfccs = melSpectrogram.map(frame => melSpectrumToMFCC(frame, mfccFeatureCount, dctMatrix))
 
 	return mfccs
 }

package/src/dsp/{MelSpectogram.ts → MelSpectrogram.ts}

@@ -2,10 +2,10 @@ import { RawAudio } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import * as FFT from './FFT.js'
 
-export async function computeMelSpectogram(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbankCount: number, lowerFrequencyHz: number, upperFrequencyHz: number, windowType: FFT.WindowType = 'hann') {
+export async function computeMelSpectrogram(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbankCount: number, lowerFrequencyHz: number, upperFrequencyHz: number, windowType: FFT.WindowType = 'hann') {
 	const logger = new Logger()
 
-	logger.start('Compute mel filterbank')
+	logger.start('Compute Mel filterbank')
 	const binCount = (fftOrder / 2) + 2
 	const nyquistFrequency = rawAudio.sampleRate / 2
 	const binFrequencies = FFT.getBinFrequencies(binCount, nyquistFrequency)
@@ -18,27 +18,28 @@ export async function computeMelSpectogram(rawAudio: RawAudio, fftOrder: number,
 
 	logger.end()
 
-	return computeMelSpectogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, melFilterbanks, windowType)
+	return computeMelSpectrogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, melFilterbanks, windowType)
 }
 
-export async function computeMelSpectogramUsingFilterbanks(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbanks: Filterbank[], windowType: FFT.WindowType = 'hann') {
+export async function computeMelSpectrogramUsingFilterbanks(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbanks: Filterbank[], windowType: FFT.WindowType = 'hann') {
 	const logger = new Logger()
 
 	logger.start('Compute short-time FFTs')
 	const audioSamples = rawAudio.audioChannels[0]
 	const fftFrames = await FFT.stftr(audioSamples, fftOrder, windowSize, hopLength, windowType)
 
-	logger.start('Convert FFT frames to a mel spectogram')
-	const melSpectogram = fftFramesToMelSpectogram(fftFrames, filterbanks)
+	logger.start('Convert FFT frames to Mel spectrogram')
+	const melSpectrogram = fftFramesToMelSpectrogram(fftFrames, filterbanks)
 
 	logger.end()
 
-	return { melSpectogram, fftFrames }
+	return { melSpectrogram, fftFrames }
 }
 
-export function fftFramesToMelSpectogram(fftFrames: Float32Array[], melFilterbanks: Filterbank[]) {
+export function fftFramesToMelSpectrogram(fftFrames: Float32Array[], melFilterbanks: Filterbank[]) {
 	return fftFrames.map(fftFrame => {
 		const powerSpectrum = FFT.fftFrameToPowerSpectrum(fftFrame)
+
 		return powerSpectrumToMelSpectrum(powerSpectrum, melFilterbanks)
 	})
 }

package/src/math/VectorMath.ts

@@ -643,7 +643,7 @@ export function sumVector(vector: ArrayLike<number>) {
 	return result
 }
 
-export function sumOfSquaresForVector(vector: ArrayLike<number>) {
+export function sumOfSquaresOfVector(vector: ArrayLike<number>) {
 	const elementCount = vector.length
 
 	let result = 0.0
@@ -655,6 +655,22 @@ export function sumOfSquaresForVector(vector: ArrayLike<number>) {
 	return result
 }
 
+export function sumAndSumOfSquaresOfVector(vector: ArrayLike<number>) {
+	const elementCount = vector.length
+
+	let sum = 0.0
+	let sumOfSquares = 0.0
+
+	for (let i = 0; i < elementCount; i++) {
+		const value = vector[i]
+
+		sum += value
+		sumOfSquares += value ** 2
+	}
+
+	return { sum, sumOfSquares }
+}
+
 export function dotProduct(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
 	if (vector1.length != vector2.length) {
 		throw new Error('Vectors are not the same length')

package/src/recognition/WhisperSTT.ts

@@ -1,9 +1,10 @@
+import chalk from 'chalk'
 import type * as Onnx from 'onnxruntime-node'
 
 import { Logger } from '../utilities/Logger.js'
-import { computeMelSpectogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectogram.js'
+import { computeMelSpectrogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectrogram.js'
 import { clip, getIntegerRange, getTopKIndexes, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
-import { indexOfMax, logOfVector, logSumExp, meanOfVector, medianOfVector, softmax, sumOfSquaresForVector, sumVector } from '../math/VectorMath.js'
+import { indexOfMax, logOfVector, logSumExp, meanOfVector, medianOfVector, softmax, sumAndSumOfSquaresOfVector, sumOfSquaresOfVector, sumVector } from '../math/VectorMath.js'
 
 import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
@@ -14,7 +15,6 @@ import { readFileAsUtf8 } from '../utilities/FileSystem.js'
 import { logLevelGreaterOrEqualTo, type LanguageDetectionResults } from '../api/API.js'
 import { formatLanguageCodeWithName, getShortLanguageCode, languageCodeToName } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
-import chalk from 'chalk'
 import { XorShift32PRNG } from '../utilities/RandomGenerator.js'
 import { detectSpeechLanguageByParts } from '../api/SpeechLanguageDetection.js'
 import { type Tiktoken } from 'tiktoken/lite'
@@ -33,7 +33,8 @@ export async function recognize(
 	modelDir: string,
 	task: WhisperTask,
 	sourceLanguage: string,
-	options: WhisperOptions) {
+	options: WhisperOptions,
+	onPart?: WhisperPartCallback) {
 
 	options = extendDeep(defaultWhisperOptions, options)
 
@@ -82,7 +83,7 @@ export async function recognize(
 		decoderProviders,
 		seed)
 
-	const result = await whisper.recognize(sourceRawAudio, task, sourceLanguage, options)
+	const result = await whisper.recognize(sourceRawAudio, task, sourceLanguage, options, undefined, onPart)
 
 	return result
 }
@@ -366,6 +367,7 @@ export class Whisper {
 		language: string,
 		options: WhisperOptions,
 		logitFilter?: WhisperLogitFilter,
+		onPart?: WhisperPartCallback,
 	) {
 		await this.initializeIfNeeded()
 
@@ -500,6 +502,13 @@ export class Whisper {
 			// Generate timeline from alignment path
 			const partTimeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, partTokens, segmentStartTime, segmentEndTime, partTokensConfidence)
 
+			if (onPart) {
+				const partWordTimeline = this.tokenTimelineToWordTimeline(partTimeline, language)
+				const partTranscript = this.tokensToText(partTokens)
+
+				onPart(partTranscript, partTimeline, partWordTimeline)
+			}
+
 			// Add tokens to output
 			allDecodedTokens.push(...partTokens)
 			timeline.push(...partTimeline)
@@ -1234,8 +1243,7 @@ export class Whisper {
 			throw new Error(`Audio part is longer than 30 seconds`)
 		}
 
-		// Compute a mel spectogram
-		await logger.startAsync('Extract mel spectogram from audio part')
+		await logger.startAsync('Extract Mel spectrogram from audio part')
 
 		// Pad audio samples to ensure that have a duration of 30 seconds
 		const paddedAudioSamples = new Float32Array(maxAudioSamples)
@@ -1243,40 +1251,40 @@ export class Whisper {
 
 		const rawAudioPart: RawAudio = { audioChannels: [paddedAudioSamples], sampleRate }
 
-		const { melSpectogram } = await computeMelSpectogramUsingFilterbanks(rawAudioPart, fftOrder, fftWindowSize, fftHopLength, filterbanks)
+		// Compute Mel spectrogram
+		const { melSpectrogram } = await computeMelSpectrogramUsingFilterbanks(rawAudioPart, fftOrder, fftWindowSize, fftHopLength, filterbanks)
 
-		await logger.startAsync('Normalize mel spectogram')
+		// Flatten, transpose, apply logarithm and normalize Mel spectrogram
+		await logger.startAsync('Process Mel spectrogram')
 
-		const logMelSpectogram = melSpectogram.map(spectrum => spectrum.map(mel => Math.log10(Math.max(mel, 1e-10))))
+		const flattenedLogMelSpectrogram = new Float32Array(maxAudioFrames * filterbankCount)
 
-		// Find maximum log mel value in the spectrum
 		let maxLogMel = -Infinity
 
-		for (const spectrum of logMelSpectogram) {
-			for (const mel of spectrum) {
-				if (mel > maxLogMel) {
-					maxLogMel = mel
+		for (let i = 0; i < filterbankCount; i++) {
+			for (let j = 0; j < maxAudioFrames; j++) {
+				const mel = melSpectrogram[j][i]
+				const logMel = Math.log10(Math.max(mel, 1e-10))
+
+				if (logMel > maxLogMel) {
+					maxLogMel = logMel
 				}
+
+				flattenedLogMelSpectrogram[(i * maxAudioFrames) + j] = logMel
 			}
 		}
 
-		// Normalize log mel spectogram (based on Python reference code)
-		const normalizedLogMelSpectogram = logMelSpectogram.map(spectrum => spectrum.map(
-			logMel => (Math.max(logMel, maxLogMel - 8) + 4) / 4))
+		for (let i = 0; i < flattenedLogMelSpectrogram.length; i++) {
+			const logMel = flattenedLogMelSpectrogram[i]
+			const normalizedLogMel = (Math.max(logMel, maxLogMel - 8) + 4) / 4
 
-		// Flatten the normalized log mel spectogram
-		const flattenedNormalizedLogMelSpectogram = new Float32Array(maxAudioFrames * filterbankCount)
-
-		for (let i = 0; i < filterbankCount; i++) {
-			for (let j = 0; j < maxAudioFrames; j++) {
-				flattenedNormalizedLogMelSpectogram[(i * maxAudioFrames) + j] = normalizedLogMelSpectogram[j][i]
-			}
+			flattenedLogMelSpectrogram[i] = normalizedLogMel
 		}
 
 		// Run the encoder model
-		await logger.startAsync('Encode mel spectogram with Whisper encoder model')
+		await logger.startAsync('Encode Mel spectrogram with Whisper encoder model')
 
-		const inputTensor = new Onnx.Tensor('float32', flattenedNormalizedLogMelSpectogram, [1, filterbankCount, maxAudioFrames])
+		const inputTensor = new Onnx.Tensor('float32', flattenedLogMelSpectrogram, [1, filterbankCount, maxAudioFrames])
 
 		const encoderInputs = { mel: inputTensor }
 
@@ -1488,8 +1496,10 @@ export class Whisper {
 				let countOfAllWeightsForHead = 0
 
 				for (const tokenFrames of head) {
-					sumOfAllWeightsForHead += sumVector(tokenFrames)
-					sumOfAllSquaredWeightsForHead += sumOfSquaresForVector(tokenFrames)
+					const { sum, sumOfSquares } = sumAndSumOfSquaresOfVector(tokenFrames)
+
+					sumOfAllWeightsForHead += sum
+					sumOfAllSquaredWeightsForHead += sumOfSquares
 					countOfAllWeightsForHead += tokenFrames.length
 				}
 
@@ -2004,7 +2014,20 @@ export type WhisperTokenData = {
 
 export type WhisperLogitFilter = (logits: number[], decodedTokens: number[], isFirstPart: boolean, isFinalPart: boolean) => number[]
 
-export type WhisperModelName = 'tiny' | 'tiny.en' | 'base' | 'base.en' | 'small' | 'small.en' | 'medium' | 'medium.en' | 'large-v1' | 'large-v2' | 'large-v3' | 'large-v3-turbo'
+export type WhisperModelName =
+	'tiny' |
+	'tiny.en' |
+	'base' |
+	'base.en' |
+	'small' |
+	'small.en' |
+	'medium' |
+	'medium.en' |
+	'large-v1' |
+	'large-v2' |
+	'large-v3' |
+	'large-v3-turbo'
+
 export type WhisperTask = 'transcribe' | 'translate' | 'detect-language'
 
 export const modelNameToPackageName: { [modelName in WhisperModelName]: string } = {
@@ -2124,17 +2147,18 @@ const languageIdLookup: { [s: string]: number } = {
 	'ba': 96,
 	'jw': 97,
 	'su': 98,
+	//'yue': 99
 }
 
 const alignmentHeadsIndexes: { [name in WhisperModelName]: number[] } = {
-	'tiny.en': [6, 12, 17, 18, 19, 20, 21, 22,],
 	'tiny': [14, 18, 20, 21, 22, 23,],
-	'base.en': [27, 39, 41, 45, 47,],
+	'tiny.en': [6, 12, 17, 18, 19, 20, 21, 22,],
 	'base': [25, 34, 35, 39, 41, 42, 44, 46,],
-	'small.en': [78, 84, 87, 92, 98, 101, 103, 108, 112, 116, 118, 120, 121, 122, 123, 126, 131, 134, 136,],
+	'base.en': [27, 39, 41, 45, 47,],
 	'small': [63, 69, 96, 100, 103, 104, 108, 115, 117, 125,],
-	'medium.en': [180, 225, 236, 238, 244, 256, 260, 265, 284, 286, 295, 298, 303, 320, 323, 329, 334, 348,],
+	'small.en': [78, 84, 87, 92, 98, 101, 103, 108, 112, 116, 118, 120, 121, 122, 123, 126, 131, 134, 136,],
 	'medium': [223, 244, 255, 257, 320, 372,],
+	'medium.en': [180, 225, 236, 238, 244, 256, 260, 265, 284, 286, 295, 298, 303, 320, 323, 329, 334, 348,],
 	'large-v1': [199, 222, 224, 237, 447, 451, 457, 462, 475,],
 	'large-v2': [212, 277, 331, 332, 333, 355, 356, 364, 371, 379, 391, 422, 423, 443, 449, 452, 465, 467, 473, 505, 521, 532, 555,],
 	'large-v3': [140, 217, 258, 272, 321, 354, 391, 424, 481, 506,],
@@ -2653,4 +2677,6 @@ export const defaultWhisperVADOptions: WhisperVADOptions = {
 	decoderProvider: undefined,
 }
 
-type WhisperTimestampAccuracy = 'medium' | 'high'
+export type WhisperTimestampAccuracy = 'medium' | 'high'
+
+export type WhisperPartCallback = (partTranscript: string, partTokenTimeline: Timeline, partWordTimeline: Timeline) => void

package/src/source-separation/MDXNetSourceSeparation.ts

@@ -1,6 +1,6 @@
 import type * as Onnx from 'onnxruntime-node'
 import { getEmptyRawAudio, RawAudio } from '../audio/AudioUtilities.js'
-import { getWindowWeights, stftrGenerator, stiftr, WindowType } from '../dsp/FFT.js'
+import { getWindowWeights, createStftrGenerator, stiftr, WindowType } from '../dsp/FFT.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { OnnxExecutionProvider, dmlProviderAvailable, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js'
@@ -74,8 +74,8 @@ export class MDXNet {
 		const fftSizeReciprocal = 1 / fftSize
 
 		// Initialize generators for STFT frames for each channel
-		const fftFramesLeftGenerator = stftrGenerator(rawAudio.audioChannels[0], fftSize, fftWindowSize, fftHopSize, fftWindowType)
-		const fftFramesRightGenerator = stftrGenerator(rawAudio.audioChannels[1], fftSize, fftWindowSize, fftHopSize, fftWindowType)
+		const fftFramesLeftGenerator = await createStftrGenerator(rawAudio.audioChannels[0], fftSize, fftWindowSize, fftHopSize, fftWindowType)
+		const fftFramesRightGenerator = await createStftrGenerator(rawAudio.audioChannels[1], fftSize, fftWindowSize, fftHopSize, fftWindowType)
 
 		// Initial windowed lists to store recently computed STFT frames
 		const fftFramesLeftWindowedList = new WindowedList<Float32Array>(segmentSize)
@@ -95,13 +95,13 @@ export class MDXNet {
 			}
 
 			while (fftFramesLeftWindowedList.endOffset < segmentEndFrameOffset) {
-				const nextLeftFrameResult = await fftFramesLeftGenerator.next()
+				const nextLeftFrameResult = fftFramesLeftGenerator.next()
 
 				if (nextLeftFrameResult.done) {
 					break
 				}
 
-				const nextRightFrameResult = await fftFramesRightGenerator.next()
+				const nextRightFrameResult = fftFramesRightGenerator.next()
 
 				if (nextRightFrameResult.done) {
 					break