echogarden 1.4.3 → 1.5.0

package/src/api/TextLanguageDetection.ts

@@ -0,0 +1,116 @@
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+import { Logger } from '../utilities/Logger.js'
+
+import * as API from './API.js'
+import { logToStderr } from '../utilities/Utilities.js'
+import { languageCodeToName } from '../utilities/Locale.js'
+import { LanguageDetectionResults } from './LanguageDetectionCommon.js'
+
+const log = logToStderr
+
+export async function detectTextLanguage(input: string, options: TextLanguageDetectionOptions): Promise<TextLanguageDetectionResult> {
+	const logger = new Logger()
+
+	options = extendDeep(defaultTextLanguageDetectionOptions, options)
+
+	const defaultLanguage = options.defaultLanguage!
+	const fallbackThresholdProbability = options.fallbackThresholdProbability!
+
+	let detectedLanguageProbabilities: LanguageDetectionResults
+
+	logger.start(`Initialize ${options.engine} module`)
+
+	switch (options.engine) {
+		case 'tinyld': {
+			const { detectLanguage } = await import('../text-language-detection/TinyLDLanguageDetection.js')
+
+			logger.start('Detect text language using tinyld')
+
+			detectedLanguageProbabilities = await detectLanguage(input)
+
+			break
+		}
+
+		case 'fasttext': {
+			const { detectLanguage } = await import('../text-language-detection/FastTextLanguageDetection.js')
+
+			logger.start('Detect text language using FastText')
+
+			detectedLanguageProbabilities = await detectLanguage(input)
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	let detectedLanguage: string
+
+	if (detectedLanguageProbabilities.length == 0 ||
+		detectedLanguageProbabilities[0].probability < fallbackThresholdProbability) {
+
+		detectedLanguage = defaultLanguage
+	} else {
+		detectedLanguage = detectedLanguageProbabilities[0].language
+	}
+
+	logger.end()
+
+	return {
+		detectedLanguage,
+		detectedLanguageName: languageCodeToName(detectedLanguage),
+		detectedLanguageProbabilities
+	}
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Types
+/////////////////////////////////////////////////////////////////////////////////////////////
+
+export interface TextLanguageDetectionResult {
+	detectedLanguage: string
+	detectedLanguageName: string
+	detectedLanguageProbabilities: LanguageDetectionResults
+}
+
+export type LanguageDetectionGroupResults = LanguageDetectionGroupResultsEntry[]
+export interface LanguageDetectionGroupResultsEntry {
+	languageGroup: string
+	probability: number
+}
+
+export type TextLanguageDetectionEngine = 'tinyld' | 'fasttext'
+
+export interface TextLanguageDetectionOptions {
+	engine?: TextLanguageDetectionEngine
+	defaultLanguage?: string
+	fallbackThresholdProbability?: number
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Constants
+/////////////////////////////////////////////////////////////////////////////////////////////
+
+export const defaultTextLanguageDetectionOptions: TextLanguageDetectionOptions = {
+	engine: 'tinyld',
+	defaultLanguage: 'en',
+	fallbackThresholdProbability: 0.05,
+}
+
+export const textLanguageDetectionEngines: API.EngineMetadata[] = [
+	{
+		id: 'tinyld',
+		name: 'TinyLD',
+		description: 'A simple language detection library.',
+		type: 'local'
+	},
+	{
+		id: 'fasttext',
+		name: 'FastText',
+		description: 'A library for word representations and sentence classification by Facebook research.',
+		type: 'local'
+	},
+]

package/src/api/TextTranslation.ts

@@ -0,0 +1,9 @@
+export async function translateText(input: string, options: TextTranslationOptions): Promise<TextTranslationResult> {
+	return {}
+}
+
+export interface TextTranslationOptions {
+}
+
+export interface TextTranslationResult {
+}

package/src/api/TimelineTranslationAlignment.ts

@@ -0,0 +1,162 @@
+import chalk from 'chalk'
+import { AudioSourceParam, RawAudio, ensureRawAudio } from '../audio/AudioUtilities.js'
+import { SubtitlesConfig } from '../subtitles/Subtitles.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, parseLangIdentifier } from '../utilities/Locale.js'
+import { Logger } from '../utilities/Logger.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
+import * as API from './API.js'
+
+export async function alignTimelineTranslation(inputTimeline: Timeline, translatedTranscript: string, options: TimelineTranslationAlignmentOptions): Promise<TimelineTranslationAlignmentResult> {
+	const logger = new Logger()
+
+	const startTimestamp = logger.getTimestamp()
+
+	options = extendDeep(defaultTimelineTranslationAlignmentOptions, options)
+
+	let rawAudio: RawAudio | undefined
+
+	if (options.audio) {
+		rawAudio = await ensureRawAudio(options.audio)
+	}
+
+	let sourceLanguage = options.sourceLanguage
+
+	if (options.sourceLanguage) {
+		const languageData = await parseLangIdentifier(options.sourceLanguage)
+
+		sourceLanguage = languageData.Name
+
+		logger.end()
+		logger.logTitledMessage('Source language specified', formatLanguageCodeWithName(sourceLanguage))
+	} else {
+		logger.start('No source language specified. Detect source language')
+
+		const timelineText = inputTimeline.map(entry => entry.text).join(' ')
+		const { detectedLanguage } = await API.detectTextLanguage(timelineText, options.languageDetection || {})
+
+		sourceLanguage = detectedLanguage
+
+		logger.end()
+		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
+	}
+
+	let targetLanguage: string
+
+	if (options.targetLanguage) {
+		const languageData = await parseLangIdentifier(options.targetLanguage)
+
+		targetLanguage = languageData.Name
+
+		logger.end()
+		logger.logTitledMessage('Target language specified', formatLanguageCodeWithName(targetLanguage))
+	} else {
+		logger.start('No target language specified. Detect target language')
+		const { detectedLanguage } = await API.detectTextLanguage(translatedTranscript, options.languageDetection || {})
+
+		targetLanguage = detectedLanguage
+
+		logger.end()
+		logger.logTitledMessage('Target language detected', formatLanguageCodeWithName(detectedLanguage))
+	}
+
+	logger.log(`Load ${options.engine} module`)
+
+	let mappedWordTimeline: Timeline
+
+	switch (options.engine) {
+		case 'e5': {
+			const { alignTimelineToTextSemantically, e5SupportedLanguages } = await import('../alignment/SemanticTextAlignment.js')
+
+			const shortSourceLanguageCode = getShortLanguageCode(sourceLanguage)
+			if (!e5SupportedLanguages.includes(shortSourceLanguageCode)) {
+				throw new Error(`Source language ${formatLanguageCodeWithName(sourceLanguage)} is not supported by the E5 embedding model.`)
+			}
+
+			const shortTargetLanguageCode = getShortLanguageCode(targetLanguage)
+			if (!e5SupportedLanguages.includes(shortTargetLanguageCode)) {
+				throw new Error(`Target language ${formatLanguageCodeWithName(targetLanguage)} is not supported by the E5 embedding model.`)
+			}
+
+			logger.end()
+
+			mappedWordTimeline = await alignTimelineToTextSemantically(
+				inputTimeline,
+				translatedTranscript,
+				targetLanguage)
+
+			break
+		}
+
+		default: {
+			throw new Error(`Unsupported engine: ${options.engine}`)
+		}
+	}
+
+	logger.start(`Postprocess timeline`)
+
+	addWordTextOffsetsToTimeline(mappedWordTimeline, translatedTranscript)
+
+	const { segmentTimeline: mappedTimeline } = await wordTimelineToSegmentSentenceTimeline(mappedWordTimeline, translatedTranscript, targetLanguage)
+
+	logger.end()
+	logger.logDuration(`Total timeline translation alignment time`, startTimestamp, chalk.magentaBright)
+
+	logger.end()
+
+	return {
+		timeline: mappedTimeline,
+		wordTimeline: mappedWordTimeline,
+
+		sourceLanguage,
+		targetLanguage,
+
+		rawAudio,
+	}
+}
+
+// Types
+export interface TimelineTranslationAlignmentResult {
+	timeline: Timeline
+	wordTimeline: Timeline
+
+	sourceLanguage?: string
+	targetLanguage: string
+
+	rawAudio?: RawAudio
+}
+
+export interface TimelineTranslationAlignmentOptions {
+	engine?: 'e5'
+
+	sourceLanguage?: string
+	targetLanguage?: string
+
+	audio?: AudioSourceParam
+
+	languageDetection?: API.TextLanguageDetectionOptions
+
+	subtitles?: SubtitlesConfig
+
+	e5?: {
+		model: 'small-fp16'
+	}
+}
+
+// Constants
+const defaultTimelineTranslationAlignmentOptions: TimelineTranslationAlignmentOptions = {
+	engine: 'e5',
+
+	sourceLanguage: undefined,
+	targetLanguage: undefined,
+
+	audio: undefined,
+
+	languageDetection: undefined,
+
+	subtitles: undefined,
+
+	e5: {
+		model: 'small-fp16',
+	}
+}

package/src/api/TranscriptAndTranslationAlignment.ts

@@ -0,0 +1,164 @@
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+import { logToStderr } from '../utilities/Utilities.js'
+import { AudioSourceParam, RawAudio } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
+
+import * as API from './API.js'
+import { Timeline } from '../utilities/Timeline.js'
+import chalk from 'chalk'
+import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
+
+const log = logToStderr
+
+export async function alignTranscriptAndTranslation(input: AudioSourceParam, transcript: string, translatedTranscript: string, options: TranscriptAndTranslationAlignmentOptions): Promise<TranscriptAndTranslationAlignmentResult> {
+	const logger = new Logger()
+
+	const startTimestamp = logger.getTimestamp()
+
+	options = extendDeep(defaultTranscriptAndTranslationAlignmentOptions, options)
+
+	if (options.sourceLanguage && !options.alignment?.language) {
+		options.alignment = extendDeep(options.alignment || {}, { language: options.sourceLanguage })
+	}
+
+	if (options.targetLanguage && !options.timelineAlignment?.targetLanguage) {
+		options.timelineAlignment = extendDeep(options.timelineAlignment || {}, { targetLanguage: options.targetLanguage })
+	}
+
+	let alignmentResult: API.AlignmentResult
+	let timelineAlignmentResult: API.TimelineTranslationAlignmentResult
+
+	switch (options.engine) {
+		case 'two-stage': {
+			logger.logTitledMessage(`Start stage 1`, `Align speech to transcript`, chalk.magentaBright)
+			logger.end()
+
+			alignmentResult = await API.align(input, transcript, options.alignment || {})
+
+			logger.log(``)
+			logger.logTitledMessage(`Start stage 2`, `Align timeline to translated transcript`, chalk.magentaBright)
+			logger.end()
+
+			timelineAlignmentResult = await API.alignTimelineTranslation(alignmentResult.timeline, translatedTranscript, options.timelineAlignment || {})
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	logger.end()
+
+	logger.log(``)
+	logger.logDuration(`Total transcript and translation alignment time`, startTimestamp, chalk.magentaBright)
+
+	return {
+		timeline: alignmentResult.timeline,
+		wordTimeline: alignmentResult.wordTimeline,
+
+		translatedTimeline: timelineAlignmentResult.timeline,
+		translatedWordTimeline: timelineAlignmentResult.wordTimeline,
+
+		transcript,
+		translatedTranscript,
+
+		sourceLanguage: alignmentResult.language,
+		targetLanguage: timelineAlignmentResult.targetLanguage,
+
+		inputRawAudio: alignmentResult.inputRawAudio,
+		isolatedRawAudio: alignmentResult.isolatedRawAudio,
+		backgroundRawAudio: alignmentResult.backgroundRawAudio,
+	}
+}
+
+export interface TranscriptAndTranslationAlignmentResult {
+	timeline: Timeline
+	wordTimeline: Timeline
+
+	translatedTimeline: Timeline
+	translatedWordTimeline: Timeline
+
+	transcript: string
+	translatedTranscript: string
+
+	sourceLanguage: string
+	targetLanguage: string
+
+	inputRawAudio: RawAudio
+	isolatedRawAudio?: RawAudio
+	backgroundRawAudio?: RawAudio
+}
+
+export type TranscriptAndTranslationAlignmentEngine = 'two-stage'
+
+export interface TranscriptAndTranslationAlignmentOptions {
+	engine?: TranscriptAndTranslationAlignmentEngine
+
+	sourceLanguage?: string
+	targetLanguage?: string
+
+	isolate?: boolean
+
+	crop?: boolean
+
+	alignment?: API.AlignmentOptions
+
+	timelineAlignment?: API.TimelineTranslationAlignmentOptions
+
+	languageDetection?: API.TextLanguageDetectionOptions
+
+	vad?: API.VADOptions
+
+	plainText?: API.PlainTextOptions
+
+	subtitles?: SubtitlesConfig
+
+	sourceSeparation?: API.SourceSeparationOptions
+}
+
+export const defaultTranscriptAndTranslationAlignmentOptions: TranscriptAndTranslationAlignmentOptions = {
+	engine: 'two-stage',
+
+	sourceLanguage: undefined,
+	targetLanguage: undefined,
+
+	isolate: false,
+
+	crop: true,
+
+	alignment: {
+	},
+
+	timelineAlignment: {
+	},
+
+	languageDetection: {
+	},
+
+	plainText: {
+		paragraphBreaks: 'double',
+		whitespace: 'collapse'
+	},
+
+	subtitles: {
+	},
+
+	vad: {
+		engine: 'adaptive-gate'
+	},
+
+	sourceSeparation: {
+	},
+}
+
+export const TranscriptAndTranslationAlignmentEngines: API.EngineMetadata[] = [
+	{
+		id: 'two-stage',
+		name: 'Two-stage translation alignment',
+		description: 'Applies two-stage translation alignment to the spoken audio. First stage aligns the speech to the native language transcript. Second stage aligns the resulting timeline with the translated transcript.',
+		type: 'local'
+	}
+]

package/src/api/TranslationAlignment.ts

@@ -6,14 +6,14 @@ import { Logger } from '../utilities/Logger.js'
 
 import * as API from './API.js'
 import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
-import { formatLanguageCodeWithName, getShortLanguageCode, normalizeIdentifierToLangaugeCode, parseLangIdentifier } from '../utilities/Locale.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeIdentifierToLanguageCode, parseLangIdentifier } from '../utilities/Locale.js'
 import { type WhisperAlignmentOptions } from '../recognition/WhisperSTT.js'
 import chalk from 'chalk'
 import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 
 const log = logToStderr
 
-export async function alignTranslation(input: AudioSourceParam, transcript: string, options: TranslationAlignmentOptions): Promise<TranslationAlignmentResult> {
+export async function alignTranslation(input: AudioSourceParam, translatedTranscript: string, options: TranslationAlignmentOptions): Promise<TranslationAlignmentResult> {
 	const logger = new Logger()
 
 	const startTimestamp = logger.getTimestamp()
@@ -77,7 +77,7 @@ export async function alignTranslation(input: AudioSourceParam, transcript: stri
 		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
 	}
 
-	const targetLanguage = await normalizeIdentifierToLangaugeCode(options.targetLanguage!)
+	const targetLanguage = await normalizeIdentifierToLanguageCode(options.targetLanguage!)
 
 	logger.logTitledMessage('Target language', formatLanguageCodeWithName(targetLanguage))
 
@@ -108,7 +108,7 @@ export async function alignTranslation(input: AudioSourceParam, transcript: stri
 				throw new Error('Whisper translation tasks are only possible with a multilingual model')
 			}
 
-			mappedTimeline = await WhisperSTT.alignEnglishTranslation(sourceRawAudio, transcript, modelName, modelDir, shortSourceLanguageCode, whisperAlignmnentOptions)
+			mappedTimeline = await WhisperSTT.alignEnglishTranslation(sourceRawAudio, translatedTranscript, modelName, modelDir, shortSourceLanguageCode, whisperAlignmnentOptions)
 
 			break
 		}
@@ -124,10 +124,10 @@ export async function alignTranslation(input: AudioSourceParam, transcript: stri
 	}
 
 	// Add text offsets
-	addWordTextOffsetsToTimeline(mappedTimeline, transcript)
+	addWordTextOffsetsToTimeline(mappedTimeline, translatedTranscript)
 
 	// Make segment timeline
-	const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(mappedTimeline, transcript, sourceLanguage, options.plainText?.paragraphBreaks, options.plainText?.whitespace)
+	const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(mappedTimeline, translatedTranscript, sourceLanguage, options.plainText?.paragraphBreaks, options.plainText?.whitespace)
 
 	logger.end()
 	logger.logDuration(`Total translation alignment time`, startTimestamp, chalk.magentaBright)
@@ -136,8 +136,9 @@ export async function alignTranslation(input: AudioSourceParam, transcript: stri
 		timeline: segmentTimeline,
 		wordTimeline: mappedTimeline,
 
-		transcript,
-		language: sourceLanguage,
+		translatedTranscript,
+		sourceLanguage,
+		targetLanguage,
 
 		inputRawAudio,
 		isolatedRawAudio,
@@ -149,8 +150,9 @@ export interface TranslationAlignmentResult {
 	timeline: Timeline
 	wordTimeline: Timeline
 
-	transcript: string
-	language: string
+	translatedTranscript: string
+	sourceLanguage: string
+	targetLanguage: string
 
 	inputRawAudio: RawAudio
 	isolatedRawAudio?: RawAudio

package/src/api/VoiceActivityDetection.ts

@@ -1,6 +1,6 @@
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 
-import { logToStderr, roundToDigits } from '../utilities/Utilities.js'
+import { logToStderr } from '../utilities/Utilities.js'
 import { AudioSourceParam, RawAudio, cropToTimeline, ensureRawAudio, } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 
@@ -220,7 +220,14 @@ export function convertCroppedToUncroppedTimeline(timeline: Timeline, uncropTime
 	}
 }
 
-function mapUsingUncropTimeline(startTimeInCroppedAudio: number, endTimeInCroppedAudio: number, uncropTimeline: Timeline) {
+function mapUsingUncropTimeline(startTimeInCroppedAudio: number, endTimeInCroppedAudio: number, uncropTimeline: Timeline): UncropTimelineMapResult {
+	if (uncropTimeline.length === 0) {
+		return {
+			mappedStartTime: 0,
+			mappedEndTime: 0,
+		}
+	}
+
 	let offsetInCroppedAudio = 0
 
 	if (endTimeInCroppedAudio < startTimeInCroppedAudio) {
@@ -250,7 +257,16 @@ function mapUsingUncropTimeline(startTimeInCroppedAudio: number, endTimeInCroppe
 	}
 
 	if (bestOverlapDuration === -1) {
-		throw new Error(`No match found in uncrop timeline (should not occur)`)
+		if (startTimeInCroppedAudio >= offsetInCroppedAudio) {
+			const maxTimestamp = uncropTimeline[uncropTimeline.length - 1].endTime
+
+			return {
+				mappedStartTime: maxTimestamp,
+				mappedEndTime: maxTimestamp
+			}
+		} else {
+			throw new Error(`Given start time ${startTimeInCroppedAudio} was smaller than audio duration but no match was found in uncrop timeline (should not occur)`)
+		}
 	}
 
 	return {
@@ -259,6 +275,11 @@ function mapUsingUncropTimeline(startTimeInCroppedAudio: number, endTimeInCroppe
 	}
 }
 
+interface UncropTimelineMapResult {
+	mappedStartTime: number
+	mappedEndTime: number
+}
+
 export interface VADResult {
 	timeline: Timeline
 	verboseTimeline: Timeline