echogarden 0.11.12 → 0.11.13

package/src/api/LanguageDetection.ts

@@ -0,0 +1,286 @@
+import { deepClone, extendDeep } from "../utilities/ObjectUtilities.js"
+
+import { AudioSourceParam, RawAudio, ensureRawAudio, getRawAudioDuration, normalizeAudioLevel, sliceRawAudioByTime, trimAudioEnd } from "../audio/AudioUtilities.js"
+import { Logger } from "../utilities/Logger.js"
+
+import * as API from "./API.js"
+import { logToStderr } from "../utilities/Utilities.js"
+import path from "path"
+import { type WhisperOptions } from "../recognition/WhisperSTT.js"
+import { formatLanguageCodeWithName, languageCodeToName } from "../utilities/Locale.js"
+import { loadPackage } from "../utilities/PackageManager.js"
+import chalk from "chalk"
+
+const log = logToStderr
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Speech language detection
+/////////////////////////////////////////////////////////////////////////////////////////////
+export async function detectSpeechLanguage(input: AudioSourceParam, options: SpeechLanguageDetectionOptions): Promise<SpeechLanguageDetectionResult> {
+	const logger = new Logger()
+
+	const startTime = logger.getTimestamp()
+
+	logger.start("Prepare for speech language detection")
+
+	const inputRawAudio = await ensureRawAudio(input)
+
+	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
+	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
+	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
+
+	options = extendDeep(defaultSpeechLanguageDetectionOptions, options)
+
+	const defaultLanguage = options.defaultLanguage!
+	const fallbackThresholdProbability = options.fallbackThresholdProbability!
+
+	logger.start(`Initialize ${options.engine} module`)
+
+	let detectedLanguageProbabilities: LanguageDetectionResults
+
+	switch (options.engine) {
+		case "silero": {
+			const SileroLanguageDetection = await import("../speech-language-detection/SileroLanguageDetection.js")
+
+			logger.end()
+
+			const sileroOptions = options.silero!
+
+			const modelDir = await loadPackage("silero-lang-classifier-95")
+
+			const modelPath = path.join(modelDir, "lang_classifier_95.onnx")
+			const languageDictionaryPath = path.join(modelDir, "lang_dict_95.json")
+			const languageGroupDictionaryPath = path.join(modelDir, "lang_group_dict_95.json")
+
+			const languageResults = await SileroLanguageDetection.detectLanguage(
+				sourceRawAudio,
+				modelPath,
+				languageDictionaryPath,
+				languageGroupDictionaryPath)
+
+			detectedLanguageProbabilities = languageResults
+
+			break
+		}
+
+		case "whisper": {
+			const WhisperSTT = await import("../recognition/WhisperSTT.js")
+
+			const whisperOptions = options.whisper!
+
+			const { modelName, modelDir, tokenizerDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, undefined)
+
+			logger.end()
+
+			detectedLanguageProbabilities = await WhisperSTT.detectLanguage(sourceRawAudio, modelName, modelDir, tokenizerDir)
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	let detectedLanguage: string
+
+	if (detectedLanguageProbabilities.length == 0 ||
+		detectedLanguageProbabilities[0].probability < fallbackThresholdProbability) {
+
+		detectedLanguage = defaultLanguage
+	} else {
+		detectedLanguage = detectedLanguageProbabilities[0].language
+	}
+
+	logger.end()
+	logger.logDuration("\nTotal detection time", startTime, chalk.magentaBright)
+
+	return { detectedLanguage, detectedLanguageName: languageCodeToName(detectedLanguage), detectedLanguageProbabilities, inputRawAudio }
+}
+
+export interface SpeechLanguageDetectionResult {
+	detectedLanguage: string
+	detectedLanguageName: string
+	detectedLanguageProbabilities: LanguageDetectionResults
+	inputRawAudio: RawAudio
+}
+
+export async function detectSpeechLanguageByParts(sourceRawAudio: RawAudio, getResultsForAudioPart: (audioPart: RawAudio) => Promise<LanguageDetectionResults>, audioPartDuration = 30, hopDuration = 15) {
+	const logger = new Logger()
+
+	const audioDuration = getRawAudioDuration(sourceRawAudio)
+
+	const resultsForParts: LanguageDetectionResults[] = []
+
+	for (let audioTimeOffset = 0; audioTimeOffset < audioDuration; audioTimeOffset += hopDuration) {
+		const startOffset = audioTimeOffset
+		const endOffset = Math.min(audioTimeOffset + audioPartDuration, audioDuration)
+		const audioPartLength = endOffset - startOffset
+
+		logger.logTitledMessage(`\nDetecting speech language starting at audio offset`, `${startOffset.toFixed(1)}`, chalk.magentaBright)
+		const audioPart = sliceRawAudioByTime(sourceRawAudio, startOffset, endOffset)
+
+		const resultsForPart = await getResultsForAudioPart(audioPart)
+
+		resultsForParts.push(resultsForPart)
+
+		const sortedResultsForPart = deepClone(resultsForPart).sort((a, b) => b.probability - a.probability)
+
+		logger.logTitledMessage(`Top candidates`, `${formatLanguageCodeWithName(sortedResultsForPart[0].language)}: ${sortedResultsForPart[0].probability.toFixed(3)}, ${formatLanguageCodeWithName(sortedResultsForPart[1].language)}: ${sortedResultsForPart[1].probability.toFixed(3)}, ${formatLanguageCodeWithName(sortedResultsForPart[3].language)}: ${sortedResultsForPart[3].probability.toFixed(3)}`)
+
+		if (audioPartLength < audioPartDuration) {
+			break
+		}
+	}
+
+	const averagedResults: LanguageDetectionResults = deepClone(resultsForParts[0])
+	averagedResults.forEach(entry => { entry.probability = 0.0 })
+
+	for (const partResults of resultsForParts) {
+		for (let i = 0; i < partResults.length; i++) {
+			averagedResults[i].probability += partResults[i].probability
+		}
+	}
+
+	for (const result of averagedResults) {
+		result.probability /= resultsForParts.length
+	}
+
+	return averagedResults
+}
+
+export type SpeechLanguageDetectionEngine = "silero" | "whisper"
+
+export interface SpeechLanguageDetectionOptions {
+	engine?: SpeechLanguageDetectionEngine
+	defaultLanguage?: string,
+	fallbackThresholdProbability?: number
+
+	silero?: {
+	}
+
+	whisper?: WhisperOptions
+}
+
+export const defaultSpeechLanguageDetectionOptions: SpeechLanguageDetectionOptions = {
+	engine: "silero",
+
+	silero: {
+	},
+
+	whisper: {
+		model: "tiny",
+	}
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Text language detection
+/////////////////////////////////////////////////////////////////////////////////////////////
+export async function detectTextLanguage(input: string, options: TextLanguageDetectionOptions): Promise<TextLanguageDetectionResult> {
+	const logger = new Logger()
+
+	options = extendDeep(defaultTextLanguageDetectionOptions, options)
+
+	const defaultLanguage = options.defaultLanguage!
+	const fallbackThresholdProbability = options.fallbackThresholdProbability!
+
+	let detectedLanguageProbabilities: LanguageDetectionResults
+
+	logger.start(`Initialize ${options.engine} module`)
+
+	switch (options.engine) {
+		case "tinyld": {
+			const { detectLanguage } = await import("../text-language-detection/TinyLDLanguageDetection.js")
+
+			logger.start("Detecting text language using tinyld")
+
+			detectedLanguageProbabilities = await detectLanguage(input)
+
+			break
+		}
+
+		case "fasttext": {
+			const { detectLanguage } = await import("../text-language-detection/FastTextLanguageDetection.js")
+
+			logger.start("Detecting text language using FastText")
+
+			detectedLanguageProbabilities = await detectLanguage(input)
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	let detectedLanguage: string
+
+	if (detectedLanguageProbabilities.length == 0 ||
+		detectedLanguageProbabilities[0].probability < fallbackThresholdProbability) {
+
+		detectedLanguage = defaultLanguage
+	} else {
+		detectedLanguage = detectedLanguageProbabilities[0].language
+	}
+
+	logger.end()
+
+	return { detectedLanguage, detectedLanguageName: languageCodeToName(detectedLanguage), detectedLanguageProbabilities }
+}
+
+export interface TextLanguageDetectionResult {
+	detectedLanguage: string
+	detectedLanguageName: string
+	detectedLanguageProbabilities: LanguageDetectionResults
+}
+
+export type LanguageDetectionResults = LanguageDetectionResultsEntry[]
+export type LanguageDetectionResultsEntry = { language: string, languageName: string, probability: number }
+
+export type LanguageDetectionGroupResults = LanguageDetectionGroupResultsEntry[]
+export type LanguageDetectionGroupResultsEntry = { languageGroup: string, probability: number }
+
+export type TextLanguageDetectionEngine = "tinyld" | "fasttext"
+
+export type TextLanguageDetectionOptions = {
+	engine?: TextLanguageDetectionEngine,
+	defaultLanguage?: string,
+	fallbackThresholdProbability?: number
+}
+
+export const defaultTextLanguageDetectionOptions: TextLanguageDetectionOptions = {
+	engine: "tinyld",
+	defaultLanguage: "en",
+	fallbackThresholdProbability: 0.05
+}
+
+export const speechLanguageDetectionEngines: API.EngineMetadata[] = [
+	{
+		id: 'silero',
+		name: 'Silero',
+		description: 'A speech language classification model by Silero.',
+		type: 'local'
+	},
+	{
+		id: 'whisper',
+		name: 'OpenAI Whisper',
+		description: 'Uses the language token produced by the Whisper model to guess the language of the speech (first 30 seconds only).',
+		type: 'local'
+	},
+]
+
+export const textLanguageDetectionEngines: API.EngineMetadata[] = [
+	{
+		id: 'tinyld',
+		name: 'TinyLD',
+		description: 'A simple language detection library.',
+		type: 'local'
+	},
+	{
+		id: 'fasttext',
+		name: 'FastText',
+		description: 'a library for word representations and sentence classification by Facebook research.',
+		type: 'local'
+	},
+]

package/src/api/Recognition.ts

@@ -0,0 +1,344 @@
+import { extendDeep } from "../utilities/ObjectUtilities.js"
+
+import { logToStderr } from "../utilities/Utilities.js"
+import { AudioSourceParam, RawAudio, ensureRawAudio, normalizeAudioLevel, trimAudioEnd } from "../audio/AudioUtilities.js"
+import { Logger } from "../utilities/Logger.js"
+
+import * as API from "./API.js"
+import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from "../utilities/Timeline.js"
+import { whisperOptionsDefaults, type WhisperOptions } from "../recognition/WhisperSTT.js"
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from "../utilities/Locale.js"
+import { loadPackage } from "../utilities/PackageManager.js"
+import chalk from "chalk"
+import { SubtitlesConfig, defaultSubtitlesBaseConfig } from "../subtitles/Subtitles.js"
+
+const log = logToStderr
+
+export async function recognize(input: AudioSourceParam, options: RecognitionOptions): Promise<RecognitionResult> {
+	const logger = new Logger()
+	const startTimestamp = logger.getTimestamp()
+
+	logger.start("Prepare for recognition")
+
+	const inputRawAudio = await ensureRawAudio(input)
+
+	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
+	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
+	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
+
+	options = extendDeep(defaultRecognitionOptions, options)
+
+	const engine = options.engine!
+
+	if (!options.language) { // && options.engine != "whisper") {
+		logger.start("No language specified. Detecting speech language")
+		const { detectedLanguage } = await API.detectSpeechLanguage(inputRawAudio, options.languageDetection || {})
+
+		logger.end()
+		logger.logTitledMessage('Language detected', formatLanguageCodeWithName(detectedLanguage))
+
+		options.language = detectedLanguage
+	}
+
+	let language = normalizeLanguageCode(options.language)
+
+	let transcript: string
+	let timeline: Timeline | undefined
+
+	logger.start(`Load ${engine} module`)
+
+	switch (engine) {
+		case "whisper": {
+			const WhisperSTT = await import("../recognition/WhisperSTT.js")
+
+			const whisperOptions = options.whisper!
+
+			const shortLanguageCode = getShortLanguageCode(language)
+
+			const { modelName, modelDir, tokenizerDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, shortLanguageCode)
+
+			if (shortLanguageCode != "en" && modelName.endsWith(".en")) {
+				throw new Error(`The model '${modelName}' is English only and cannot transcribe language '${shortLanguageCode}'`)
+			}
+
+			logger.end();
+
+			({ transcript, timeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, tokenizerDir, "transcribe", language, whisperOptions))
+
+			break
+		}
+
+		case "vosk": {
+			const VoskSTT = await import("../recognition/VoskSTT.js")
+
+			try {
+				await import('@echogarden/vosk')
+			} catch (e) {
+				log(e)
+				throw new Error(`The vosk npm package, which is required for Vosk support, was not found, or had an error loading. If missing, you can install it by running 'npm install @echogarden/vosk -g'.`)
+			}
+
+			const voskOptions = options.vosk!
+
+			const modelPath = voskOptions.modelPath
+
+			if (!modelPath) {
+				throw new Error("Vosk models are not currently auto-downloaded. You'll need to download a model manually and set a model path in 'vosk.modelPath'.")
+			}
+
+			logger.end();
+
+			({ transcript, timeline } = await VoskSTT.recognize(sourceRawAudio, modelPath, true))
+
+			break
+		}
+
+		case "silero": {
+			const SileroSTT = await import("../recognition/SileroSTT.js")
+
+			const sileroOptions = options.silero!
+
+			let modelPath = sileroOptions.modelPath
+
+			if (!modelPath) {
+				const shortLanguageCode = getShortLanguageCode(language)
+				const packageName = SileroSTT.languageCodeToPackageName[shortLanguageCode]
+
+				if (!packageName) {
+					throw new Error(`Language '${shortLanguageCode}' is not supported by Silero`)
+				}
+
+				modelPath = await loadPackage(packageName)
+			}
+
+			logger.end();
+
+			({ transcript, timeline } = await SileroSTT.recognize(sourceRawAudio, modelPath))
+
+			break
+		}
+
+		case "google-cloud": {
+			const GoogleCloudSTT = await import("../recognition/GoogleCloudSTT.js")
+
+			const apiKey = options.googleCloud!.apiKey
+
+			if (!apiKey) {
+				throw new Error(`No API key given`)
+			}
+
+			logger.end();
+
+			({ transcript, timeline } = await GoogleCloudSTT.recognize(sourceRawAudio, apiKey, language))
+
+			break
+		}
+
+		case "microsoft-azure": {
+			const AzureCognitiveServicesSTT = await import("../recognition/AzureCognitiveServicesSTT.js")
+
+			const subscriptionKey = options.microsoftAzure!.subscriptionKey
+
+			if (!subscriptionKey) {
+				throw new Error(`No subscription key given`)
+			}
+
+			const serviceRegion = options.microsoftAzure!.serviceRegion
+
+			if (!serviceRegion) {
+				throw new Error(`No service region given`)
+			}
+
+			logger.end();
+
+			({ transcript, timeline } = await AzureCognitiveServicesSTT.recognize(sourceRawAudio, subscriptionKey, serviceRegion, language))
+
+			break
+		}
+
+		case "amazon-transcribe": {
+			const AmazonTranscribeSTT = await import("../recognition/AmazonTranscribeSTT.js")
+
+			const region = options.amazonTranscribe!.region
+
+			if (!region) {
+				throw new Error(`No region given`)
+			}
+
+			const accessKeyId = options.amazonTranscribe!.accessKeyId
+
+			if (!accessKeyId) {
+				throw new Error(`No access key id given`)
+			}
+
+			const secretAccessKey = options.amazonTranscribe!.secretAccessKey
+
+			if (!secretAccessKey) {
+				throw new Error(`No secret access key given`)
+			}
+
+			logger.end();
+
+			({ transcript, timeline } = await AmazonTranscribeSTT.recgonize(sourceRawAudio, language, region, accessKeyId, secretAccessKey))
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	if (!timeline) {
+		logger.start(`Align audio to transcript`)
+		const alignmentOptions: API.AlignmentOptions = extendDeep(options.alignment, { language: language })
+
+		const { wordTimeline } = await API.align(sourceRawAudio, transcript, alignmentOptions)
+
+		timeline = wordTimeline
+	}
+
+	addWordTextOffsetsToTimeline(timeline, transcript)
+
+	const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(timeline, transcript, language, 'single', 'preserve')
+
+	logger.end()
+	logger.logDuration('Total recognition time', startTimestamp, chalk.magentaBright)
+
+	return { transcript, timeline: segmentTimeline, wordTimeline: timeline, inputRawAudio, language }
+}
+
+export interface RecognitionResult {
+	transcript: string
+	timeline: Timeline
+	wordTimeline: Timeline
+	language: string
+	inputRawAudio: RawAudio
+}
+
+export type RecognitionEngine = "whisper" | "vosk" | "silero" | "google-cloud" | "microsoft-azure" | "amazon-transcribe"
+
+export interface RecognitionOptions {
+	engine?: RecognitionEngine
+
+	language?: string
+
+	maxAlternatives?: number
+
+	alignment?: API.AlignmentOptions
+
+	languageDetection?: API.SpeechLanguageDetectionOptions
+
+	subtitles?: SubtitlesConfig
+
+	whisper?: WhisperOptions
+
+	vosk?: {
+		modelPath?: string
+	}
+
+	silero?: {
+		modelPath?: string
+	}
+
+	googleCloud?: {
+		apiKey?: string
+		alternativeLanguageCodes?: string[]
+		profanityFilter?: boolean
+		autoPunctuation?: boolean
+		useEnhancedModel?: boolean
+	}
+
+	microsoftAzure?: {
+		subscriptionKey?: string
+		serviceRegion?: string
+	}
+
+	amazonTranscribe?: {
+		region?: string
+		accessKeyId?: string
+		secretAccessKey?: string
+	}
+}
+
+export const defaultRecognitionOptions: RecognitionOptions = {
+	engine: "whisper",
+
+	language: undefined,
+
+	maxAlternatives: 1,
+
+	alignment: undefined,
+
+	languageDetection: undefined,
+
+	subtitles: defaultSubtitlesBaseConfig,
+
+	whisper: whisperOptionsDefaults,
+
+	vosk: {
+		modelPath: undefined
+	},
+
+	silero: {
+		modelPath: undefined
+	},
+
+	googleCloud: {
+		apiKey: undefined,
+		alternativeLanguageCodes: [],
+		profanityFilter: false,
+		autoPunctuation: true,
+		useEnhancedModel: true,
+	},
+
+	microsoftAzure: {
+		subscriptionKey: undefined,
+		serviceRegion: undefined
+	},
+
+	amazonTranscribe: {
+		region: undefined,
+		accessKeyId: undefined,
+		secretAccessKey: undefined,
+	}
+}
+
+export const recognitionEngines: API.EngineMetadata[] = [
+	{
+		id: 'whisper',
+		name: 'OpenAI Whisper',
+		description: 'A high accuracy transformer-based architecture by OpenAI.',
+		type: 'local'
+	},
+	{
+		id: 'vosk',
+		name: 'Vosk',
+		description: 'A speech recognition toolkit.',
+		type: 'local'
+	},
+	{
+		id: 'silero',
+		name: 'Silero',
+		description: 'Speech recognition models.',
+		type: 'local'
+	},
+	{
+		id: 'google-cloud',
+		name: 'Google Cloud',
+		description: 'Google Cloud speech-to-text service.',
+		type: 'cloud'
+	},
+	{
+		id: 'microsoft-azure',
+		name: 'Azure Cognitive Services',
+		description: 'Microsoft Azure speech-to-text service.',
+		type: 'cloud'
+	},
+	{
+		id: 'amazon-transcribe',
+		name: 'Amazon Transcribe',
+		description: 'Amazon cloud speech-to-text service.',
+		type: 'cloud'
+	},
+]