echogarden 0.12.2 → 1.0.0

package/src/api/LanguageDetection.ts

@@ -1,15 +1,16 @@
-import { deepClone, extendDeep } from "../utilities/ObjectUtilities.js"
+import { deepClone, extendDeep } from '../utilities/ObjectUtilities.js'
 
-import { AudioSourceParam, RawAudio, ensureRawAudio, getRawAudioDuration, normalizeAudioLevel, sliceRawAudioByTime, trimAudioEnd } from "../audio/AudioUtilities.js"
-import { Logger } from "../utilities/Logger.js"
+import { AudioSourceParam, RawAudio, ensureRawAudio, getRawAudioDuration, normalizeAudioLevel, sliceRawAudioByTime, trimAudioEnd } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
 
-import * as API from "./API.js"
-import { logToStderr } from "../utilities/Utilities.js"
-import path from "path"
-import { type WhisperOptions } from "../recognition/WhisperSTT.js"
-import { formatLanguageCodeWithName, languageCodeToName } from "../utilities/Locale.js"
-import { loadPackage } from "../utilities/PackageManager.js"
-import chalk from "chalk"
+import * as API from './API.js'
+import { logToStderr } from '../utilities/Utilities.js'
+import path from 'path'
+import { WhisperModelName } from '../recognition/WhisperSTT.js'
+import { formatLanguageCodeWithName, languageCodeToName } from '../utilities/Locale.js'
+import { loadPackage } from '../utilities/PackageManager.js'
+import chalk from 'chalk'
+import { WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
 
 const log = logToStderr
 
@@ -21,7 +22,7 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 
 	const startTime = logger.getTimestamp()
 
-	logger.start("Prepare for speech language detection")
+	options = extendDeep(defaultSpeechLanguageDetectionOptions, options)
 
 	const inputRawAudio = await ensureRawAudio(input)
 
@@ -29,7 +30,14 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
 
-	options = extendDeep(defaultSpeechLanguageDetectionOptions, options)
+	if (options.crop) {
+		logger.start('Crop using voice activity detection');
+		({ croppedRawAudio: sourceRawAudio } = await API.detectVoiceActivity(sourceRawAudio, options.vad!))
+
+		logger.end()
+	}
+
+	logger.start('Prepare for speech language detection')
 
 	const defaultLanguage = options.defaultLanguage!
 	const fallbackThresholdProbability = options.fallbackThresholdProbability!
@@ -39,18 +47,18 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 	let detectedLanguageProbabilities: LanguageDetectionResults
 
 	switch (options.engine) {
-		case "silero": {
-			const SileroLanguageDetection = await import("../speech-language-detection/SileroLanguageDetection.js")
+		case 'silero': {
+			const SileroLanguageDetection = await import('../speech-language-detection/SileroLanguageDetection.js')
 
 			logger.end()
 
 			const sileroOptions = options.silero!
 
-			const modelDir = await loadPackage("silero-lang-classifier-95")
+			const modelDir = await loadPackage('silero-lang-classifier-95')
 
-			const modelPath = path.join(modelDir, "lang_classifier_95.onnx")
-			const languageDictionaryPath = path.join(modelDir, "lang_dict_95.json")
-			const languageGroupDictionaryPath = path.join(modelDir, "lang_group_dict_95.json")
+			const modelPath = path.join(modelDir, 'lang_classifier_95.onnx')
+			const languageDictionaryPath = path.join(modelDir, 'lang_dict_95.json')
+			const languageGroupDictionaryPath = path.join(modelDir, 'lang_group_dict_95.json')
 
 			const languageResults = await SileroLanguageDetection.detectLanguage(
 				sourceRawAudio,
@@ -63,16 +71,32 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 			break
 		}
 
-		case "whisper": {
-			const WhisperSTT = await import("../recognition/WhisperSTT.js")
+		case 'whisper': {
+			const WhisperSTT = await import('../recognition/WhisperSTT.js')
 
 			const whisperOptions = options.whisper!
 
-			const { modelName, modelDir, tokenizerDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, undefined)
+			const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, undefined)
+
+			logger.end()
+
+			detectedLanguageProbabilities = await WhisperSTT.detectLanguage(sourceRawAudio, modelName, modelDir, whisperOptions.temperature!)
+
+			break
+		}
+
+		case 'whisper.cpp': {
+			const WhisperCppSTT = await import('../recognition/WhisperCppSTT.js')
+
+			const whisperCppOptions = options.whisperCpp!
 
 			logger.end()
 
-			detectedLanguageProbabilities = await WhisperSTT.detectLanguage(sourceRawAudio, modelName, modelDir, tokenizerDir)
+			const { modelName, modelPath } = await WhisperCppSTT.loadModelPackage(whisperCppOptions.model, undefined)
+
+			logger.end();
+
+			detectedLanguageProbabilities = await WhisperCppSTT.detectLanguage(sourceRawAudio, modelName, modelPath)
 
 			break
 		}
@@ -93,9 +117,15 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 	}
 
 	logger.end()
-	logger.logDuration("\nTotal detection time", startTime, chalk.magentaBright)
+	logger.logDuration('\nTotal detection time', startTime, chalk.magentaBright)
+
+	return {
+		detectedLanguage,
+		detectedLanguageName: languageCodeToName(detectedLanguage),
+		detectedLanguageProbabilities,
 
-	return { detectedLanguage, detectedLanguageName: languageCodeToName(detectedLanguage), detectedLanguageProbabilities, inputRawAudio }
+		inputRawAudio,
+	}
 }
 
 export interface SpeechLanguageDetectionResult {
@@ -105,11 +135,15 @@ export interface SpeechLanguageDetectionResult {
 	inputRawAudio: RawAudio
 }
 
-export async function detectSpeechLanguageByParts(sourceRawAudio: RawAudio, getResultsForAudioPart: (audioPart: RawAudio) => Promise<LanguageDetectionResults>, audioPartDuration = 30, hopDuration = 15) {
+export async function detectSpeechLanguageByParts(sourceRawAudio: RawAudio, getResultsForAudioPart: (audioPart: RawAudio) => Promise<LanguageDetectionResults>, audioPartDuration = 30, hopDuration = 25) {
 	const logger = new Logger()
 
 	const audioDuration = getRawAudioDuration(sourceRawAudio)
 
+	if (audioDuration === 0) {
+		return []
+	}
+
 	const resultsForParts: LanguageDetectionResults[] = []
 
 	for (let audioTimeOffset = 0; audioTimeOffset < audioDuration; audioTimeOffset += hopDuration) {
@@ -126,7 +160,13 @@ export async function detectSpeechLanguageByParts(sourceRawAudio: RawAudio, getR
 
 		const sortedResultsForPart = deepClone(resultsForPart).sort((a, b) => b.probability - a.probability)
 
-		logger.logTitledMessage(`Top candidates`, `${formatLanguageCodeWithName(sortedResultsForPart[0].language)}: ${sortedResultsForPart[0].probability.toFixed(3)}, ${formatLanguageCodeWithName(sortedResultsForPart[1].language)}: ${sortedResultsForPart[1].probability.toFixed(3)}, ${formatLanguageCodeWithName(sortedResultsForPart[3].language)}: ${sortedResultsForPart[3].probability.toFixed(3)}`)
+		let topCandidatesStrings: string[] = []
+
+		for (let i = 0; i < Math.min(3, sortedResultsForPart.length); i++) {
+			topCandidatesStrings.push(`${formatLanguageCodeWithName(sortedResultsForPart[i].language)}: ${sortedResultsForPart[i].probability.toFixed(3)}`)
+		}
+
+		logger.logTitledMessage(`Top candidates`, topCandidatesStrings.join(', '))
 
 		if (audioPartLength < audioPartDuration) {
 			break
@@ -149,27 +189,49 @@ export async function detectSpeechLanguageByParts(sourceRawAudio: RawAudio, getR
 	return averagedResults
 }
 
-export type SpeechLanguageDetectionEngine = "silero" | "whisper"
+export type SpeechLanguageDetectionEngine = 'silero' | 'whisper' | 'whisper.cpp'
 
 export interface SpeechLanguageDetectionOptions {
 	engine?: SpeechLanguageDetectionEngine
 	defaultLanguage?: string,
 	fallbackThresholdProbability?: number
 
+	crop?: boolean
+
 	silero?: {
 	}
 
-	whisper?: WhisperOptions
+	whisper?: {
+		model?: WhisperModelName
+		temperature?: number
+	}
+
+	whisperCpp?: WhisperCppOptions
+
+	vad?: API.VADOptions
 }
 
 export const defaultSpeechLanguageDetectionOptions: SpeechLanguageDetectionOptions = {
-	engine: "silero",
+	engine: 'whisper',
+	defaultLanguage: 'en',
+	fallbackThresholdProbability: 0.05,
+
+	crop: true,
 
 	silero: {
 	},
 
 	whisper: {
-		model: "tiny",
+		model: 'tiny',
+		temperature: 1.0
+	},
+
+	whisperCpp: {
+		model: 'tiny'
+	},
+
+	vad: {
+		engine: 'adaptive-gate'
 	}
 }
 
@@ -189,20 +251,20 @@ export async function detectTextLanguage(input: string, options: TextLanguageDet
 	logger.start(`Initialize ${options.engine} module`)
 
 	switch (options.engine) {
-		case "tinyld": {
-			const { detectLanguage } = await import("../text-language-detection/TinyLDLanguageDetection.js")
+		case 'tinyld': {
+			const { detectLanguage } = await import('../text-language-detection/TinyLDLanguageDetection.js')
 
-			logger.start("Detecting text language using tinyld")
+			logger.start('Detecting text language using tinyld')
 
 			detectedLanguageProbabilities = await detectLanguage(input)
 
 			break
 		}
 
-		case "fasttext": {
-			const { detectLanguage } = await import("../text-language-detection/FastTextLanguageDetection.js")
+		case 'fasttext': {
+			const { detectLanguage } = await import('../text-language-detection/FastTextLanguageDetection.js')
 
-			logger.start("Detecting text language using FastText")
+			logger.start('Detecting text language using FastText')
 
 			detectedLanguageProbabilities = await detectLanguage(input)
 
@@ -226,9 +288,17 @@ export async function detectTextLanguage(input: string, options: TextLanguageDet
 
 	logger.end()
 
-	return { detectedLanguage, detectedLanguageName: languageCodeToName(detectedLanguage), detectedLanguageProbabilities }
+	return {
+		detectedLanguage,
+		detectedLanguageName: languageCodeToName(detectedLanguage),
+		detectedLanguageProbabilities
+	}
 }
 
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Types
+/////////////////////////////////////////////////////////////////////////////////////////////
+
 export interface TextLanguageDetectionResult {
 	detectedLanguage: string
 	detectedLanguageName: string
@@ -236,23 +306,34 @@ export interface TextLanguageDetectionResult {
 }
 
 export type LanguageDetectionResults = LanguageDetectionResultsEntry[]
-export type LanguageDetectionResultsEntry = { language: string, languageName: string, probability: number }
+export interface LanguageDetectionResultsEntry {
+	language: string
+	languageName: string
+	probability: number
+}
 
 export type LanguageDetectionGroupResults = LanguageDetectionGroupResultsEntry[]
-export type LanguageDetectionGroupResultsEntry = { languageGroup: string, probability: number }
+export interface LanguageDetectionGroupResultsEntry {
+	languageGroup: string
+	probability: number
+}
 
-export type TextLanguageDetectionEngine = "tinyld" | "fasttext"
+export type TextLanguageDetectionEngine = 'tinyld' | 'fasttext'
 
-export type TextLanguageDetectionOptions = {
-	engine?: TextLanguageDetectionEngine,
-	defaultLanguage?: string,
+export interface TextLanguageDetectionOptions {
+	engine?: TextLanguageDetectionEngine
+	defaultLanguage?: string
 	fallbackThresholdProbability?: number
 }
 
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Constants
+/////////////////////////////////////////////////////////////////////////////////////////////
+
 export const defaultTextLanguageDetectionOptions: TextLanguageDetectionOptions = {
-	engine: "tinyld",
-	defaultLanguage: "en",
-	fallbackThresholdProbability: 0.05
+	engine: 'tinyld',
+	defaultLanguage: 'en',
+	fallbackThresholdProbability: 0.05,
 }
 
 export const speechLanguageDetectionEngines: API.EngineMetadata[] = [
@@ -265,7 +346,13 @@ export const speechLanguageDetectionEngines: API.EngineMetadata[] = [
 	{
 		id: 'whisper',
 		name: 'OpenAI Whisper',
-		description: 'Uses the language token produced by the Whisper model to guess the language of the speech (first 30 seconds only).',
+		description: 'Uses the language tokens produced by the Whisper model to classify the spoken langauge.',
+		type: 'local'
+	},
+	{
+		id: 'whisper.cpp',
+		name: 'OpenAI Whisper (C++ port)',
+		description: 'Uses the language tokens produced by Whisper.cpp to classify the spoken langauge.',
 		type: 'local'
 	},
 ]
@@ -280,7 +367,7 @@ export const textLanguageDetectionEngines: API.EngineMetadata[] = [
 	{
 		id: 'fasttext',
 		name: 'FastText',
-		description: 'a library for word representations and sentence classification by Facebook research.',
+		description: 'A library for word representations and sentence classification by Facebook research.',
 		type: 'local'
 	},
 ]