echogarden 1.0.4 → 1.1.0

package/src/math/VectorMath.ts

@@ -1,5 +1,3 @@
-import createMedianFilter from 'moving-median'
-
 export function covarianceMatrixOfSamples(samples: number[][], weights?: number[], biased = false) {
 	if (samples.length == 0) {
 		throw new Error('No vectors given')
@@ -507,7 +505,7 @@ export function cosineSimilarity(vector1: number[], vector2: number[]) {
 		squaredMagnitude2 += vector2[i] ** 2
 	}
 
-	const result = dotProduct / (Math.sqrt(squaredMagnitude1) * Math.sqrt(squaredMagnitude2))
+	const result = dotProduct / (Math.sqrt(squaredMagnitude1) * Math.sqrt(squaredMagnitude2) + 1e-40)
 
 	return zeroIfNaN(result)
 }
@@ -651,28 +649,6 @@ export function indexOfMin(vector: number[]) {
 	return result
 }
 
-export function exponentialSmoothingMeanOfVectors(vectors: number[][], smoothingFactor: number) {
-	const vectorCount = vectors.length
-
-	if (vectorCount == 0) {
-		return []
-	}
-
-	const featureCount = vectors[0].length
-
-	const currentEstimate = createVector(featureCount)
-
-	for (let i = 0; i < vectorCount; i++) {
-		for (let j = 0; j < featureCount; j++) {
-			const value = vectors[i][j]
-
-			currentEstimate[j] = (smoothingFactor * value) + ((1 - smoothingFactor) * currentEstimate[j])
-		}
-	}
-
-	return currentEstimate
-}
-
 export function sigmoid(x: number) {
 	const result = 1 / (1 + Math.exp(-x))
 
@@ -728,17 +704,6 @@ export function hammingDistance(value1: number, value2: number, bitLength = 32)
 	return result
 }
 
-export function medianFilter(vector: number[], width: number) {
-	const filter = createMedianFilter(width)
-	const result = []
-
-	for (let i = 0; i < vector.length; i++) {
-		result.push(filter(vector[i]))
-	}
-
-	return result
-}
-
 export function createVectorArray(vectorCount: number, featureCount: number, initialValue = 0.0) {
 	const result: number[][] = new Array(vectorCount)
 

package/src/recognition/OpenAICloudSTT.ts

@@ -81,33 +81,6 @@ class FileLikeBlob extends Blob {
 	}
 }
 
-export interface OpenAICloudSTTOptions {
-	model?: 'whisper-1'
-
-	apiKey?: string
-	organization?: string
-	baseURL?: string
-
-	temperature?: number
-	prompt?: string
-
-	timeout?: number
-	maxRetries?: number
-}
-
-export const defaultOpenAICloudSTTOptions: OpenAICloudSTTOptions = {
-	apiKey: undefined,
-	organization: undefined,
-	baseURL: undefined,
-
-	model: 'whisper-1',
-	temperature: 0,
-	prompt: undefined,
-
-	timeout: undefined,
-	maxRetries: 10,
-}
-
 interface VerboseResponse {
 	task: string
 	language: string
@@ -140,3 +113,30 @@ interface VerboseResponse {
 }
 
 type Task = 'transcribe' | 'translate'
+
+export interface OpenAICloudSTTOptions {
+	model?: 'whisper-1'
+
+	apiKey?: string
+	organization?: string
+	baseURL?: string
+
+	temperature?: number
+	prompt?: string
+
+	timeout?: number
+	maxRetries?: number
+}
+
+export const defaultOpenAICloudSTTOptions: OpenAICloudSTTOptions = {
+	apiKey: undefined,
+	organization: undefined,
+	baseURL: undefined,
+
+	model: 'whisper-1',
+	temperature: 0,
+	prompt: undefined,
+
+	timeout: undefined,
+	maxRetries: 10,
+}

package/src/recognition/SileroSTT.ts

@@ -4,167 +4,204 @@ import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { RawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
-import { readAndParseJsonFile, readFile } from '../utilities/FileSystem.js'
-import type * as Onnx from 'onnxruntime-node'
+import { readAndParseJsonFile } from '../utilities/FileSystem.js'
 import path from 'path'
 
+import type * as Onnx from 'onnxruntime-node'
+import { OnnxExecutionProvider, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js'
+
 const log = logToStderr
 
-export async function recognize(rawAudio: RawAudio, modelDirectory: string) {
-	const logger = new Logger()
-	logger.start('Create ONNX inference session')
+export async function recognize(
+	rawAudio: RawAudio,
+	modelDirectoryPath: string,
+	executionProviders: OnnxExecutionProvider[]) {
 
-	const Onnx = await import('onnxruntime-node')
+	const silero = new SileroSTT(modelDirectoryPath, executionProviders)
 
-	const modelPath = path.join(modelDirectory, 'model.onnx')
-	const labelsPath = path.join(modelDirectory, 'labels.json')
+	const result = await silero.recognize(rawAudio)
 
-	const labels: string[] = await readAndParseJsonFile(labelsPath)
+	return result
+}
+
+export class SileroSTT {
+	session?: Onnx.InferenceSession
+	labels?: string[]
 
-	const onnxOptions: Onnx.InferenceSession.SessionOptions = {
-		logSeverityLevel: 3
+	constructor(
+		public readonly modelDirectoryPath: string,
+		public readonly executionProviders: OnnxExecutionProvider[]
+	) {
 	}
 
-	const recognition = await Onnx.InferenceSession.create(modelPath, onnxOptions)
+	async recognize(rawAudio: RawAudio) {
+		const logger = new Logger()
 
-	logger.start('Prepare input data')
+		await this.initializeIfNeeded()
 
-	const audioSamples = rawAudio.audioChannels[0]
+		logger.start('Recognize with silero model')
 
-	const inputTensor = new Onnx.Tensor('float32', audioSamples, [1, audioSamples.length])
+		const audioSamples = rawAudio.audioChannels[0]
 
-	const inputs = { input: inputTensor }
+		const Onnx = await import('onnxruntime-node')
 
-	logger.start('Recognize with silero model')
+		const inputTensor = new Onnx.Tensor('float32', audioSamples, [1, audioSamples.length])
+		const inputs = { input: inputTensor }
 
-	const results = await recognition.run(inputs)
+		const results = await this.session!.run(inputs)
 
-	const rawResultValues = results['output'].data as Float32Array
+		const rawResultValues = results['output'].data as Float32Array
 
-	const tokenResults: Float32Array[] = []
+		const labels = this.labels!
 
-	for (let i = 0; i < rawResultValues.length; i += labels.length) {
-		tokenResults.push(rawResultValues.subarray(i, i + labels.length))
-	}
+		const tokenResults: Float32Array[] = []
 
-	const tokens: string[] = []
+		for (let i = 0; i < rawResultValues.length; i += labels.length) {
+			tokenResults.push(rawResultValues.subarray(i, i + labels.length))
+		}
 
-	for (const tokenResult of tokenResults) {
-		const bestCandidateIndex = indexOfMax(new Array(...tokenResult))
-		tokens.push(labels[bestCandidateIndex])
-	}
+		const tokens: string[] = []
+
+		for (const tokenResult of tokenResults) {
+			const bestCandidateIndex = indexOfMax(new Array(...tokenResult))
+			tokens.push(labels[bestCandidateIndex])
+		}
 
-	//log(tokens.join('|'))
+		//log(tokens.join('|'))
 
-	const result = processTokens(tokens, getRawAudioDuration(rawAudio))
+		const result = this.tokensToTimeline(tokens, getRawAudioDuration(rawAudio))
 
-	logger.end()
+		logger.end()
 
-	return result
-}
+		return result
+	}
 
-function processTokens(tokens: string[], totalDuration: number) {
-	const tokenCount = tokens.length
+	private async initializeIfNeeded() {
+		if (this.session) {
+			return
+		}
 
-	const decodedTokens: string[] = []
-	let tokenGroupIndexes: number[][] = [[]]
+		const logger = new Logger()
 
-	for (let i = 0; i < tokenCount; i++) {
-		const token = tokens[i]
+		logger.start('Create ONNX inference session')
 
-		if (token == '2') {
-			if (decodedTokens.length > 0) {
-				const previousDecodedToken = decodedTokens[decodedTokens.length - 1]
-				decodedTokens.push('$')
-				decodedTokens.push(previousDecodedToken)
+		const Onnx = await import('onnxruntime-node')
 
-				tokenGroupIndexes[tokenGroupIndexes.length - 1].push(i)
-			} else {
-				decodedTokens.push(' ')
-				tokenGroupIndexes.push([])
-			}
+		const modelPath = path.join(this.modelDirectoryPath, 'model.onnx')
+		const labelsPath = path.join(this.modelDirectoryPath, 'labels.json')
 
-			continue
-		}
+		this.labels = await readAndParseJsonFile(labelsPath)
 
-		if (token == '_') {
-			continue
-		}
+		const onnxSessionOptions = getOnnxSessionOptions({ executionProviders: this.executionProviders })
 
-		decodedTokens.push(token)
+		this.session = await Onnx.InferenceSession.create(modelPath, onnxSessionOptions)
 
-		if (token == ' ') {
-			tokenGroupIndexes.push([])
-		} else {
-			tokenGroupIndexes[tokenGroupIndexes.length - 1].push(i)
-		}
+		logger.end()
 	}
 
-	let decodedString = ''
-
-	for (let i = 0; i < decodedTokens.length; i++) {
-		const currentToken = decodedTokens[i]
-		const previousToken = decodedTokens[i - 1]
+	private tokensToTimeline(tokens: string[], totalDuration: number) {
+		const tokenCount = tokens.length
 
-		if (currentToken != '$' && (previousToken != currentToken || previousToken == undefined)) {
-			decodedString += currentToken
-		}
-	}
+		const decodedTokens: string[] = []
+		let tokenGroupIndexes: number[][] = [[]]
 
-	decodedString = decodedString.trim()
+		for (let i = 0; i < tokenCount; i++) {
+			const token = tokens[i]
 
-	tokenGroupIndexes = tokenGroupIndexes.filter(group => group.length > 0)
+			if (token == '2') {
+				if (decodedTokens.length > 0) {
+					const previousDecodedToken = decodedTokens[decodedTokens.length - 1]
+					decodedTokens.push('$')
+					decodedTokens.push(previousDecodedToken)
 
-	if (tokenGroupIndexes.length > 0) {
-		let currentCorrection = Math.min(tokenGroupIndexes[0][0], 1.5)
+					tokenGroupIndexes[tokenGroupIndexes.length - 1].push(i)
+				} else {
+					decodedTokens.push(' ')
+					tokenGroupIndexes.push([])
+				}
 
-		for (let i = 0; i < tokenGroupIndexes.length; i++) {
-			const group = tokenGroupIndexes[i]
+				continue
+			}
 
-			if (group.length == 1) {
-				group.push(group[0])
+			if (token == '_') {
+				continue
 			}
 
-			group[0] -= currentCorrection
+			decodedTokens.push(token)
 
-			if (i == tokenGroupIndexes.length - 1) {
-				currentCorrection = Math.min(tokenCount - i, 1.5)
+			if (token == ' ') {
+				tokenGroupIndexes.push([])
 			} else {
-				currentCorrection = Math.min((tokenGroupIndexes[i + 1][0] - group[group.length - 1]) / 2, 1.5)
+				tokenGroupIndexes[tokenGroupIndexes.length - 1].push(i)
 			}
+		}
 
-			group[group.length - 1] += currentCorrection
+		let decodedString = ''
+
+		for (let i = 0; i < decodedTokens.length; i++) {
+			const currentToken = decodedTokens[i]
+			const previousToken = decodedTokens[i - 1]
+
+			if (currentToken != '$' && (previousToken != currentToken || previousToken == undefined)) {
+				decodedString += currentToken
+			}
 		}
-	}
 
-	const words = decodedString.split(' ')
+		decodedString = decodedString.trim()
+
+		tokenGroupIndexes = tokenGroupIndexes.filter(group => group.length > 0)
+
+		if (tokenGroupIndexes.length > 0) {
+			let currentCorrection = Math.min(tokenGroupIndexes[0][0], 1.5)
 
-	const timeMultiplier = totalDuration / tokenCount
+			for (let i = 0; i < tokenGroupIndexes.length; i++) {
+				const group = tokenGroupIndexes[i]
 
-	const timeline: Timeline = []
+				if (group.length == 1) {
+					group.push(group[0])
+				}
 
-	for (let i = 0; i < words.length; i++) {
-		const text = words[i]
+				group[0] -= currentCorrection
 
-		if (!wordCharacterPattern.test(text)) {
-			continue
+				if (i == tokenGroupIndexes.length - 1) {
+					currentCorrection = Math.min(tokenCount - i, 1.5)
+				} else {
+					currentCorrection = Math.min((tokenGroupIndexes[i + 1][0] - group[group.length - 1]) / 2, 1.5)
+				}
+
+				group[group.length - 1] += currentCorrection
+			}
 		}
 
-		const group = tokenGroupIndexes[i]
-		const startTime = group[0] * timeMultiplier
-		const endTime = group[group.length - 1] * timeMultiplier
+		const words = decodedString.split(' ')
 
-		timeline.push({
-			type: 'word',
-			text: text,
-			startTime,
-			endTime,
-		})
-	}
+		const timeMultiplier = totalDuration / tokenCount
 
-	timeline[timeline.length - 1].endTime = totalDuration
+		const timeline: Timeline = []
 
-	return { transcript: decodedString, timeline }
+		for (let i = 0; i < words.length; i++) {
+			const text = words[i]
+
+			if (!wordCharacterPattern.test(text)) {
+				continue
+			}
+
+			const group = tokenGroupIndexes[i]
+			const startTime = group[0] * timeMultiplier
+			const endTime = group[group.length - 1] * timeMultiplier
+
+			timeline.push({
+				type: 'word',
+				text: text,
+				startTime,
+				endTime,
+			})
+		}
+
+		timeline[timeline.length - 1].endTime = totalDuration
+
+		return { transcript: decodedString, timeline }
+	}
 }
 
 export const languageCodeToPackageName: { [languageCode: string]: string } = {
@@ -173,3 +210,13 @@ export const languageCodeToPackageName: { [languageCode: string]: string } = {
 	'de': 'silero-de-v1',
 	'uk': 'silero-ua-v3',
 }
+
+export interface SileroRecognitionOptions {
+	modelPath?: string
+	provider?: OnnxExecutionProvider
+}
+
+export const defaultSileroRecognitionOptions: SileroRecognitionOptions = {
+	modelPath: undefined,
+	provider: undefined,
+}

package/src/recognition/WhisperCppSTT.ts

@@ -233,7 +233,7 @@ export async function detectLanguage(sourceRawAudio: RawAudio, modelName: Whispe
 async function parseResultObject(resultObject: WhisperCppVerboseResult, modelName: WhisperModelName, totalDuration: number, enableDTW: boolean): Promise<RecognitionResult> {
 	const { Whisper } = await import('../recognition/WhisperSTT.js')
 
-	const whisper = new Whisper(modelName, '')
+	const whisper = new Whisper(modelName, '', [], [])
 	await whisper.initializeTokenizerIfNeeded()
 
 	const tokenTimeline: Timeline = []