echogarden 1.4.4 → 1.6.0

package/src/dsp/KWeightingFilter.ts

@@ -10,7 +10,7 @@ export class KWeightingFilter {
 
 		if (useStandard44100Filters) {
 			// These parameter values are taken from ITU-R BS.1770-2
-			// and designed only for a 44100 Hz sampling rate:
+			// and designed only for a 44100 Hz sample rate:
 			this.highShelfFilter = new BiquadFilter({
 				b0: 1.53512485958697,
 				b1: -2.69169618940638,

package/src/dsp/MelSpectogram.ts

@@ -2,7 +2,7 @@ import { RawAudio } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import * as FFT from './FFT.js'
 
-export async function computeMelSpectogram(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbankCount: number, lowerFrequencyHz: number, upperFrequencyHz: number) {
+export async function computeMelSpectogram(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbankCount: number, lowerFrequencyHz: number, upperFrequencyHz: number, windowType: FFT.WindowType = 'hann') {
 	const logger = new Logger()
 
 	logger.start('Compute mel filterbank')
@@ -18,15 +18,15 @@ export async function computeMelSpectogram(rawAudio: RawAudio, fftOrder: number,
 
 	logger.end()
 
-	return computeMelSpectogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, melFilterbanks)
+	return computeMelSpectogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, melFilterbanks, windowType)
 }
 
-export async function computeMelSpectogramUsingFilterbanks(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbanks: Filterbank[]) {
+export async function computeMelSpectogramUsingFilterbanks(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbanks: Filterbank[], windowType: FFT.WindowType = 'hann') {
 	const logger = new Logger()
 
 	logger.start('Compute short-time FFTs')
 	const audioSamples = rawAudio.audioChannels[0]
-	const fftFrames = await FFT.stftr(audioSamples, fftOrder, windowSize, hopLength, 'hann')
+	const fftFrames = await FFT.stftr(audioSamples, fftOrder, windowSize, hopLength, windowType)
 
 	logger.start('Convert FFT frames to a mel spectogram')
 	const melSpectogram = fftFramesToMelSpectogram(fftFrames, filterbanks)
@@ -52,17 +52,26 @@ export function powerSpectrumToMelSpectrum(powerSpectrum: Float32Array, filterba
 		const filterbankStartIndex = filterbank.startIndex
 		const filterbankWeights = filterbank.weights
 
-		if (filterbankStartIndex == -1) {
+		if (filterbankStartIndex === -1) {
 			continue
 		}
 
-		let bandValue = 0
+		let melBandValue = 0
 
 		for (let i = 0; i < filterbankWeights.length; i++) {
-			bandValue += filterbankWeights[i] * powerSpectrum[filterbankStartIndex + i]
+			const powerSpectrumIndex = filterbankStartIndex + i
+
+			if (powerSpectrumIndex >= powerSpectrum.length) {
+				break
+			}
+
+			const weight = filterbankWeights[i]
+			const powerSpectrumValue = powerSpectrum[powerSpectrumIndex]
+
+			melBandValue += weight * powerSpectrumValue
 		}
 
-		melSpectrum[melBandIndex] = bandValue
+		melSpectrum[melBandIndex] = melBandValue
 	}
 
 	return melSpectrum

package/src/math/VectorMath.ts

@@ -177,7 +177,7 @@ export function scaleToSumTo1(vector: number[]) {
 	return scaledVector
 }
 
-export function normalizeVector(vector: number[], kind: 'population' | 'sample' = 'population') {
+export function normalizeVector(vector: ArrayLike<number>, kind: 'population' | 'sample' = 'population') {
 	if (vector.length == 0) {
 		throw new Error('Vector is empty')
 	}
@@ -329,15 +329,15 @@ export function varianceOfVectors(vectors: number[][], kind: 'population' | 'sam
 	return result
 }
 
-export function meanOfVector(vector: number[]) {
+export function meanOfVector(vector: ArrayLike<number>) {
 	if (vector.length == 0) {
-		throw new Error('Vector is empty')
+		return 0
 	}
 
 	return sumVector(vector) / vector.length
 }
 
-export function medianOfVector(vector: number[]) {
+export function medianOfVector(vector: ArrayLike<number>) {
 	if (vector.length == 0) {
 		throw new Error('Vector is empty')
 	}
@@ -345,13 +345,13 @@ export function medianOfVector(vector: number[]) {
 	return vector[Math.floor(vector.length / 2)]
 }
 
-export function stdDeviationOfVector(vector: number[], kind: 'population' | 'sample' = 'population', mean?: number) {
+export function stdDeviationOfVector(vector: ArrayLike<number>, kind: 'population' | 'sample' = 'population', mean?: number) {
 	return Math.sqrt(varianceOfVector(vector, kind, mean))
 }
 
-export function varianceOfVector(vector: number[], kind: 'population' | 'sample' = 'population', mean?: number) {
+export function varianceOfVector(vector: ArrayLike<number>, kind: 'population' | 'sample' = 'population', mean?: number) {
 	if (vector.length == 0) {
-		throw new Error('Vector is empty')
+		return 0
 	}
 
 	const sampleSizeMetric = kind == 'population' || vector.length == 1 ? vector.length : vector.length - 1
@@ -362,8 +362,8 @@ export function varianceOfVector(vector: number[], kind: 'population' | 'sample'
 
 	let result = 0.0
 
-	for (const value of vector) {
-		result += (value - mean) ** 2
+	for (let i = 0; i < vector.length; i++) {
+		result += (vector[i] - mean) ** 2
 	}
 
 	return result / sampleSizeMetric
@@ -456,11 +456,11 @@ export function meanSquaredError(actual: ArrayLike<number>, expected: ArrayLike<
 	return sum / featureCount
 }
 
-export function euclidianDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
-	return Math.sqrt(squaredEuclidianDistance(vector1, vector2))
+export function euclideanDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
+	return Math.sqrt(squaredEuclideanDistance(vector1, vector2))
 }
 
-export function squaredEuclidianDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
+export function squaredEuclideanDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
 	if (vector1.length !== vector2.length) {
 		throw new Error('Vectors are not the same length')
 	}
@@ -480,11 +480,11 @@ export function squaredEuclidianDistance(vector1: ArrayLike<number>, vector2: Ar
 	return sum
 }
 
-export function euclidianDistance13Dim(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
-	return Math.sqrt(squaredEuclidianDistance13Dim(vector1, vector2))
+export function euclideanDistance13Dim(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
+	return Math.sqrt(squaredEuclideanDistance13Dim(vector1, vector2))
 }
 
-export function squaredEuclidianDistance13Dim(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
+export function squaredEuclideanDistance13Dim(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
 	// Assumes the input has 13 dimensions (optimized for 13-dimensional MFCC vectors)
 
 	const result =

package/src/nlp/ChineseSegmentation.ts

@@ -1,5 +1,5 @@
 export async function splitChineseTextToWords_Jieba(text: string, fineGrained = false, useHMM = true) {
-	const jieba = await getWasmInstance()
+	const jieba = await getJiebaWasmInstance()
 
 	if (!fineGrained) {
 		return jieba.cut(text, useHMM)
@@ -58,10 +58,12 @@ export async function splitChineseTextToWords_Jieba(text: string, fineGrained =
 }
 
 let JiebaWasmInstance: typeof import('jieba-wasm')
-async function getWasmInstance() {
+
+async function getJiebaWasmInstance() {
 	if (!JiebaWasmInstance) {
-		const { default: JibeaWasm } = await import('jieba-wasm')
-		JiebaWasmInstance = JibeaWasm
+		const { default: JiebaWasm } = await import('jieba-wasm')
+
+		JiebaWasmInstance = JiebaWasm
 	}
 
 	return JiebaWasmInstance

package/src/nlp/Segmentation.ts

@@ -11,8 +11,7 @@ const log = logToStderr
 export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u
 export const punctuationPattern = /[\p{Punctuation}]/u
 
-export const phraseSeparators = [',', ';', ':']
-export const sentenceSeparators = ['.', '?', '!']
+export const phraseSeparators = [',', ';', ':', '，', '、']
 export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '=', '¼', '½', '¾']
 
 export function isWordOrSymbolWord(str: string) {
@@ -226,29 +225,35 @@ export async function splitToWords(text: string, langCode: string): Promise<stri
 	}
 }
 
-export function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakType, whitespace: WhitespaceProcessing) {
+export function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakType, whitespaceProcessingMethod: WhitespaceProcessing) {
 	let paragraphs: string[] = []
 
-	if (paragraphBreaks == 'single') {
+	if (paragraphBreaks === 'single') {
 		paragraphs = text.split(/(\r?\n)+/g)
-	} else if (paragraphBreaks == 'double') {
+	} else if (paragraphBreaks === 'double') {
 		paragraphs = text.split(/(\r?\n)(\r?\n)+/g)
 	} else {
-		throw new Error(`Invalid paragraph break type: ${paragraphBreaks}`)
+		throw new Error(`Invalid paragraph break type: '${paragraphBreaks}'`)
 	}
 
-	if (whitespace == 'removeLineBreaks') {
-		paragraphs = paragraphs.map(p => p.replaceAll(/(\r?\n)+/g, ' '))
-	} else if (whitespace == 'collapse') {
-		paragraphs = paragraphs.map(p => p.replaceAll(/\s+/g, ' '))
-	}
-
-	paragraphs = paragraphs.map(p => p.trim())
+	paragraphs = paragraphs.map(p => applyWhitespaceProcessing(p.trim(), whitespaceProcessingMethod))
 	paragraphs = paragraphs.filter(p => p.length > 0)
 
 	return paragraphs
 }
 
+export function applyWhitespaceProcessing(text: string, whitespaceProcessingMethod: WhitespaceProcessing) {
+	if (whitespaceProcessingMethod === 'removeLineBreaks') {
+		return text.replaceAll(/(\r?\n)+/g, ' ')
+	} else if (whitespaceProcessingMethod === 'collapse') {
+		return text.replaceAll(/\s+/g, ' ')
+	} else if (whitespaceProcessingMethod === 'preserve') {
+		return text
+	} else {
+		throw new Error(`Invalid whitespace processing method: '${whitespaceProcessingMethod}'`)
+	}
+}
+
 export function splitToLines(text: string) {
 	return text.split(/\r?\n/g)
 }

package/src/recognition/OpenAICloudSTT.ts

@@ -1,8 +1,10 @@
-import { RawAudio } from '../audio/AudioUtilities.js';
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
-import { Logger } from '../utilities/Logger.js';
-import { extendDeep } from '../utilities/ObjectUtilities.js';
-import { Timeline, TimelineEntry } from '../utilities/Timeline.js';
+import { RawAudio } from '../audio/AudioUtilities.js'
+import { createVirtualFileReadStreamForBuffer } from '../utilities/VirtualFileReadStream.js'
+import { Logger } from '../utilities/Logger.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
+import { alignSegments } from '../api/Alignment.js'
 
 export async function recognize(rawAudio: RawAudio, languageCode: string, options: OpenAICloudSTTOptions, task: Task = 'transcribe') {
 	const logger = new Logger()
@@ -11,41 +13,56 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 
 	options = extendDeep(defaultOpenAICloudSTTOptions, options)
 
+	if (options.requestWordTimestamps === undefined) {
+		options.requestWordTimestamps = options.baseURL === undefined
+	}
+
+	if (options.model === undefined) {
+		if (options.baseURL === undefined) {
+			options.model = 'whisper-1'
+		} else {
+			throw new Error(`A custom provider for the OpenAI Cloud API requires specifying a model name`)
+		}
+	}
+
 	const { default: OpenAI } = await import('openai')
 	const openai = new OpenAI(options)
 
 	logger.start('Encode audio to send')
 	const ffmpegOptions = FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('mp3')
 	const encodedAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, ffmpegOptions)
-	const audioAsWaveBlob = new FileLikeBlob([encodedAudio], 'audio', Date.now(), { type: 'audio/mpeg' })
+	const virtualFileStream = createVirtualFileReadStreamForBuffer(encodedAudio, 'audio.mp3')
 
-	logger.start('Request recognition from OpenAI Cloud API')
+	logger.start(options.baseURL ? `Send request to ${options.baseURL}` : 'Send request to OpenAI Cloud API')
 
 	let response: VerboseResponse
 
-	if (task =='transcribe') {
+	if (task == 'transcribe') {
+		const timestamp_granularities: ('word' | 'segment')[] | undefined =
+			options.requestWordTimestamps ? ['word', 'segment'] : undefined
+
 		response = await openai.audio.transcriptions.create({
-			file: audioAsWaveBlob,
-			model: options.model!,
+			file: virtualFileStream,
+			model: options.model,
 			language: languageCode,
 			prompt: options.prompt,
 			response_format: 'verbose_json',
 			temperature: options.temperature,
-			timestamp_granularities: ['word', 'segment']
-		}) as VerboseResponse
+			timestamp_granularities,
+		}) as any as VerboseResponse
 	} else if (task == 'translate') {
 		response = await openai.audio.translations.create({
-			file: audioAsWaveBlob,
-			model: options.model!,
+			file: virtualFileStream,
+			model: options.model,
 			prompt: options.prompt,
 			response_format: 'verbose_json',
 			temperature: options.temperature,
-		}) as VerboseResponse
+		}) as any as VerboseResponse
 	} else {
 		throw new Error(`Invalid task`)
 	}
 
-	const transcript = response.text
+	const transcript = response.text.trim()
 
 	let timeline: Timeline
 
@@ -57,12 +74,20 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			endTime: entry.end
 		}))
 	} else {
-		timeline = response.segments.map<TimelineEntry>(entry => ({
+		const segmentTimeline = response.segments.map<TimelineEntry>(entry => ({
 			type: 'segment',
 			text: entry.text,
 			startTime: entry.start,
 			endTime: entry.end
 		}))
+
+		if (task === 'transcribe') {
+			logger.start('Align segments')
+
+			timeline = await alignSegments(rawAudio, segmentTimeline, { language: languageCode })
+		} else {
+			timeline = segmentTimeline
+		}
 	}
 
 	logger.end()
@@ -70,17 +95,6 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 	return { transcript, timeline }
 }
 
-class FileLikeBlob extends Blob {
-	constructor(
-		public readonly parts: BlobPart[],
-		public readonly name: string,
-		public readonly lastModified: number,
-		options: BlobPropertyBag,
-	) {
-		super(parts, options)
-	}
-}
-
 interface VerboseResponse {
 	task: string
 	language: string
@@ -115,7 +129,7 @@ interface VerboseResponse {
 type Task = 'transcribe' | 'translate'
 
 export interface OpenAICloudSTTOptions {
-	model?: 'whisper-1'
+	model?: 'whisper-1' | string
 
 	apiKey?: string
 	organization?: string
@@ -126,6 +140,8 @@ export interface OpenAICloudSTTOptions {
 
 	timeout?: number
 	maxRetries?: number
+
+	requestWordTimestamps?: boolean
 }
 
 export const defaultOpenAICloudSTTOptions: OpenAICloudSTTOptions = {
@@ -133,10 +149,12 @@ export const defaultOpenAICloudSTTOptions: OpenAICloudSTTOptions = {
 	organization: undefined,
 	baseURL: undefined,
 
-	model: 'whisper-1',
+	model: undefined,
 	temperature: 0,
 	prompt: undefined,
 
 	timeout: undefined,
 	maxRetries: 10,
+
+	requestWordTimestamps: undefined,
 }

package/src/recognition/WhisperCppSTT.ts

@@ -13,7 +13,7 @@ import { splitToLines } from '../nlp/Segmentation.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { formatLanguageCodeWithName, getShortLanguageCode } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
-import { detectSpeechLanguageByParts } from '../api/LanguageDetection.js'
+import { detectSpeechLanguageByParts } from '../api/SpeechLanguageDetection.js'
 
 export async function recognize(
 	sourceRawAudio: RawAudio,
@@ -54,7 +54,7 @@ export async function recognize(
 				}
 			} else {
 				if (options.enableGPU) {
-					buildKind = 'cublas-11.8.0'
+					buildKind = 'cublas-12.4.0'
 				} else {
 					buildKind = 'cpu'
 				}
@@ -201,7 +201,7 @@ export async function recognize(
 
 export async function detectLanguage(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelPath: string) {
 	if (sourceRawAudio.sampleRate != 16000) {
-		throw new Error('Source audio must have a sampling rate of 16000')
+		throw new Error('Source audio must have a sample rate of 16000')
 	}
 
 	async function detectLanguageForPart(partAudio: RawAudio) {
@@ -240,6 +240,8 @@ async function parseResultObject(resultObject: WhisperCppVerboseResult, modelNam
 
 	let currentCorrectionTimeOffset = 0
 
+	let lastTokenEndOffset = 0
+
 	for (let segmentIndex = 0; segmentIndex < resultObject.transcription.length; segmentIndex++) {
 		const segmentObject = resultObject.transcription[segmentIndex]
 
@@ -248,6 +250,17 @@ async function parseResultObject(resultObject: WhisperCppVerboseResult, modelNam
 		for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex++) {
 			const tokenObject = tokens[tokenIndex]
 
+			// Workaround whisper.cpp issue with missing offsets by falling back to last known end offset
+			// when they are not included
+			if (!tokenObject.offsets) {
+				tokenObject.offsets = {
+					from: lastTokenEndOffset,
+					to: lastTokenEndOffset,
+				}
+			} else {
+				lastTokenEndOffset = tokenObject.offsets.to
+			}
+
 			if (tokenIndex === 0 && tokenObject.text === '[_BEG_]' && tokenObject.offsets.from === 0) {
 				currentCorrectionTimeOffset = segmentObject.offsets.from / 1000
 			}
@@ -370,7 +383,7 @@ export async function loadModelPackage(modelId: WhisperCppModelId | undefined, l
 	return { modelName, modelPath }
 }
 
-export type WhisperCppBuild = 'cpu' | 'cublas-11.8.0' | 'cublas-12.4.0' | 'custom'
+export type WhisperCppBuild = 'cpu' | 'cublas-12.4.0' | 'custom'
 
 export async function loadExecutablePackage(buildKind: WhisperCppBuild) {
 	if (buildKind === 'custom') {
@@ -384,20 +397,20 @@ export async function loadExecutablePackage(buildKind: WhisperCppBuild) {
 
 	if (buildKind.startsWith('cublas-')) {
 		if (platform === 'win32' && arch === 'x64') {
-			packageName = `whisper.cpp-binaries-windows-x64-${buildKind}-latest-patched`
+			packageName = `whisper.cpp-binaries-windows-x64-${buildKind}-latest`
 		} else {
-			throw new Error(`GPU builds (NVIDIA CUDA only) are currently only available as packages for Windows x64. Please specify a custom path to the binary in the 'executablePath' option.`)
+			throw new Error(`whisper.cpp GPU builds (NVIDIA CUDA only) are currently only available as packages for Windows x64. Please specify a custom path to a whisper.cpp 'main' binary in the 'executablePath' option.`)
 		}
 	} else if (buildKind === 'cpu') {
 		if (platform === 'win32' && arch === 'x64') {
-			packageName = `whisper.cpp-binaries-windows-x64-cpu-latest-patched`
+			packageName = `whisper.cpp-binaries-windows-x64-cpu-latest`
 		} else if (platform === 'linux' && arch === 'x64') {
-			packageName = `whisper.cpp-binaries-linux-x64-cpu-latest-patched`
+			packageName = `whisper.cpp-binaries-linux-x64-cpu-latest`
 		} else {
-			throw new Error(`Couldn't find a matching whisper.cpp binary package. Please specify a custom path to the binary in the 'executablePath' option.`)
+			throw new Error(`Couldn't find a matching whisper.cpp binary package. Please specify a custom path to a whisper.cpp 'main' binary in the 'executablePath' option.`)
 		}
 	} else {
-		throw new Error(`Unknown build kind '${buildKind}'`)
+		throw new Error(`Unsupported build kind '${buildKind}'`)
 	}
 
 	const packagePath = await loadPackage(packageName)
@@ -553,4 +566,6 @@ export type WhisperCppModelId =
 	'large-v2' |
 	'large-v2-q5_0' |
 	'large-v3' |
-	'large-v3-q5_0'
+	'large-v3-q5_0' |
+	`large-v3-turbo` |
+	`large-v3-turbo-q5_0`