echogarden 1.4.4 → 1.6.0

package/src/api/Synthesis.ts

@@ -15,7 +15,7 @@ import { loadLexiconsForLanguage } from '../nlp/Lexicon.js'
 import * as API from './API.js'
 import { Timeline, TimelineEntry, addTimeOffsetToTimeline, multiplyTimelineByFactor } from '../utilities/Timeline.js'
 import { getAppDataDir, ensureDir, existsSync, isFileIsUpToDate, readAndParseJsonFile, writeFileSafe } from '../utilities/FileSystem.js'
-import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode, defaultDialectForLanguageCode, parseLangIdentifier, normalizeIdentifierToLangaugeCode } from '../utilities/Locale.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode, defaultDialectForLanguageCode, parseLangIdentifier, normalizeIdentifierToLanguageCode } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import { EngineMetadata, appName } from './Common.js'
 import { shouldCancelCurrentTask } from '../server/Worker.js'
@@ -332,7 +332,7 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 	let language: string
 
 	if (options.language) {
-		language = await normalizeIdentifierToLangaugeCode(options.language)
+		language = await normalizeIdentifierToLanguageCode(options.language)
 	} else {
 		language = selectedVoice.languages[0]
 	}
@@ -1556,7 +1556,7 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 		voiceList = await loadVoiceList()
 	}
 
-	const languageCode = await normalizeIdentifierToLangaugeCode(options.language || '')
+	const languageCode = await normalizeIdentifierToLanguageCode(options.language || '')
 
 	if (languageCode) {
 		let filteredVoiceList = voiceList.filter(voice => voice.languages.includes(languageCode))
@@ -1619,7 +1619,7 @@ export interface RequestVoiceListResult {
 }
 
 export async function selectBestOfflineEngineForLanguage(language: string): Promise<SynthesisEngine> {
-	language = await normalizeIdentifierToLangaugeCode(language)
+	language = await normalizeIdentifierToLanguageCode(language)
 
 	const VitsTTS = await import('../synthesis/VitsTTS.js')
 

package/src/api/TextLanguageDetection.ts

@@ -0,0 +1,116 @@
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+import { Logger } from '../utilities/Logger.js'
+
+import * as API from './API.js'
+import { logToStderr } from '../utilities/Utilities.js'
+import { languageCodeToName } from '../utilities/Locale.js'
+import { LanguageDetectionResults } from './LanguageDetectionCommon.js'
+
+const log = logToStderr
+
+export async function detectTextLanguage(input: string, options: TextLanguageDetectionOptions): Promise<TextLanguageDetectionResult> {
+	const logger = new Logger()
+
+	options = extendDeep(defaultTextLanguageDetectionOptions, options)
+
+	const defaultLanguage = options.defaultLanguage!
+	const fallbackThresholdProbability = options.fallbackThresholdProbability!
+
+	let detectedLanguageProbabilities: LanguageDetectionResults
+
+	logger.start(`Initialize ${options.engine} module`)
+
+	switch (options.engine) {
+		case 'tinyld': {
+			const { detectLanguage } = await import('../text-language-detection/TinyLDLanguageDetection.js')
+
+			logger.start('Detect text language using tinyld')
+
+			detectedLanguageProbabilities = await detectLanguage(input)
+
+			break
+		}
+
+		case 'fasttext': {
+			const { detectLanguage } = await import('../text-language-detection/FastTextLanguageDetection.js')
+
+			logger.start('Detect text language using FastText')
+
+			detectedLanguageProbabilities = await detectLanguage(input)
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	let detectedLanguage: string
+
+	if (detectedLanguageProbabilities.length == 0 ||
+		detectedLanguageProbabilities[0].probability < fallbackThresholdProbability) {
+
+		detectedLanguage = defaultLanguage
+	} else {
+		detectedLanguage = detectedLanguageProbabilities[0].language
+	}
+
+	logger.end()
+
+	return {
+		detectedLanguage,
+		detectedLanguageName: languageCodeToName(detectedLanguage),
+		detectedLanguageProbabilities
+	}
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Types
+/////////////////////////////////////////////////////////////////////////////////////////////
+
+export interface TextLanguageDetectionResult {
+	detectedLanguage: string
+	detectedLanguageName: string
+	detectedLanguageProbabilities: LanguageDetectionResults
+}
+
+export type LanguageDetectionGroupResults = LanguageDetectionGroupResultsEntry[]
+export interface LanguageDetectionGroupResultsEntry {
+	languageGroup: string
+	probability: number
+}
+
+export type TextLanguageDetectionEngine = 'tinyld' | 'fasttext'
+
+export interface TextLanguageDetectionOptions {
+	engine?: TextLanguageDetectionEngine
+	defaultLanguage?: string
+	fallbackThresholdProbability?: number
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Constants
+/////////////////////////////////////////////////////////////////////////////////////////////
+
+export const defaultTextLanguageDetectionOptions: TextLanguageDetectionOptions = {
+	engine: 'tinyld',
+	defaultLanguage: 'en',
+	fallbackThresholdProbability: 0.05,
+}
+
+export const textLanguageDetectionEngines: API.EngineMetadata[] = [
+	{
+		id: 'tinyld',
+		name: 'TinyLD',
+		description: 'A simple language detection library.',
+		type: 'local'
+	},
+	{
+		id: 'fasttext',
+		name: 'FastText',
+		description: 'A library for word representations and sentence classification by Facebook research.',
+		type: 'local'
+	},
+]

package/src/api/TextTranslation.ts

@@ -0,0 +1,177 @@
+import chalk from 'chalk'
+import { formatLanguageCodeWithName, normalizeIdentifierToLanguageCode, parseLangIdentifier } from '../utilities/Locale.js'
+import { Logger } from '../utilities/Logger.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import * as API from './API.js'
+
+export async function translateText(inputText: string, options: TextTranslationOptions): Promise<TextTranslationResult> {
+	const logger = new Logger()
+
+	const startTimestamp = logger.getTimestamp()
+
+	options = extendDeep(defaultTextTranslationOptions, options)
+
+	if (options.sourceLanguage) {
+		const languageData = await parseLangIdentifier(options.sourceLanguage)
+
+		options.sourceLanguage = languageData.Name
+
+		logger.end()
+		logger.logTitledMessage('Source language specified', formatLanguageCodeWithName(options.sourceLanguage))
+	} else {
+		logger.start('No source language specified. Detect text language')
+		const { detectedLanguage } = await API.detectTextLanguage(inputText, options.languageDetection || {})
+
+		options.sourceLanguage = detectedLanguage
+
+		logger.end()
+		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
+	}
+
+	options.targetLanguage = await normalizeIdentifierToLanguageCode(options.targetLanguage!)
+
+	logger.logTitledMessage('Target language', formatLanguageCodeWithName(options.targetLanguage))
+
+	logger.start(`Load ${options.engine} module`)
+
+	let translationPairs: TranslationPair[]
+
+	switch (options.engine) {
+		case 'nllb': {
+			const NLLBTextTranslation = await import('../text-translation/NLLBTextTranslation.js')
+
+			logger.end()
+
+			logger.logTitledMessage(`Warning`, `The nllb engine is currently an early prototype implementation and doesn't work correctly.`, chalk.yellow, 'warning')
+
+			translationPairs = await NLLBTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage)
+
+			break
+		}
+
+		case 'google-translate': {
+			const GoogleTranslateTextTranslation = await import('../text-translation/GoogleTranslateTextTranslation.js')
+
+			logger.end()
+
+			translationPairs = await GoogleTranslateTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage)
+
+			break
+		}
+
+		case 'deepl': {
+			const DeepLTextTranslation = await import('../text-translation/DeepLTextTranslation.js')
+
+			logger.end()
+
+			logger.logTitledMessage(`Warning`, `The deepl engine is currently an early prototype implementation and doesn't work correctly.`, chalk.yellow, 'warning')
+
+			translationPairs = await DeepLTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage)
+
+			break
+		}
+
+		default: {
+			throw new Error(`'${options.engine}' is not a supported text translation engine.`)
+		}
+	}
+
+	const translatedText = translationPairs.map(pair => {
+		const translated = pair.translatedText
+
+		if (translated.endsWith(' ') || translated.endsWith('\n')) {
+			return pair.translatedText
+		} else {
+			return pair.translatedText + ' '
+		}
+	}).join('').trim()
+
+	logger.end()
+
+	logger.log('')
+	logger.logDuration(`Total text translation time`, startTimestamp, chalk.magentaBright)
+
+	return {
+		text: inputText,
+		translatedText,
+
+		translationPairs,
+
+		sourceLanguage: options.sourceLanguage!,
+		targetLanguage: options.targetLanguage!,
+	}
+}
+
+export interface TextTranslationOptions {
+	engine?: TextTranslationEngine
+
+	sourceLanguage?: string
+	targetLanguage?: string
+
+	languageDetection?: API.TextLanguageDetectionOptions
+
+	nllb?: {
+	},
+
+	googleTranslate?: {
+	},
+
+	deepl?: {
+	},
+}
+
+export interface TextTranslationResult {
+	text: string
+	translatedText: string
+
+	translationPairs: TranslationPair[]
+
+	sourceLanguage: string
+	targetLanguage: string
+}
+
+export type TextTranslationEngine = 'nllb' | 'google-translate' | 'deepl'
+
+export interface TranslationPair {
+	sourceText: string
+	translatedText: string
+}
+
+export const defaultTextTranslationOptions: TextTranslationOptions = {
+	engine: 'google-translate',
+
+	sourceLanguage: undefined,
+	targetLanguage: 'en',
+
+	languageDetection: undefined,
+
+	nllb: {
+	},
+
+	googleTranslate: {
+	},
+
+	deepl: {
+	},
+}
+
+export const textTranslationEngines: API.EngineMetadata[] = [
+	{
+		id: 'nllb',
+		name: 'NLLB',
+		description: 'No Language Left Behind (NLLB) is a deep learning machine translation model by Facebook Research (early prototype implementation).',
+		type: 'local'
+	},
+	{
+		id: 'google-translate',
+		name: 'Google Translate',
+		description: 'Unoffical text translation API used by the Google Translate web interface.',
+		type: 'cloud'
+	},
+	{
+		id: 'deepl',
+		name: 'DeepL',
+		description: 'Unoffical text translation API used by the DeepL web interface (early prototype implementation).',
+		type: 'cloud'
+	},
+]

package/src/api/TimelineTranslationAlignment.ts

@@ -0,0 +1,162 @@
+import chalk from 'chalk'
+import { AudioSourceParam, RawAudio, ensureRawAudio } from '../audio/AudioUtilities.js'
+import { SubtitlesConfig } from '../subtitles/Subtitles.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, parseLangIdentifier } from '../utilities/Locale.js'
+import { Logger } from '../utilities/Logger.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
+import * as API from './API.js'
+
+export async function alignTimelineTranslation(inputTimeline: Timeline, translatedTranscript: string, options: TimelineTranslationAlignmentOptions): Promise<TimelineTranslationAlignmentResult> {
+	const logger = new Logger()
+
+	const startTimestamp = logger.getTimestamp()
+
+	options = extendDeep(defaultTimelineTranslationAlignmentOptions, options)
+
+	let rawAudio: RawAudio | undefined
+
+	if (options.audio) {
+		rawAudio = await ensureRawAudio(options.audio)
+	}
+
+	let sourceLanguage = options.sourceLanguage
+
+	if (options.sourceLanguage) {
+		const languageData = await parseLangIdentifier(options.sourceLanguage)
+
+		sourceLanguage = languageData.Name
+
+		logger.end()
+		logger.logTitledMessage('Source language specified', formatLanguageCodeWithName(sourceLanguage))
+	} else {
+		logger.start('No source language specified. Detect source language')
+
+		const timelineText = inputTimeline.map(entry => entry.text).join(' ')
+		const { detectedLanguage } = await API.detectTextLanguage(timelineText, options.languageDetection || {})
+
+		sourceLanguage = detectedLanguage
+
+		logger.end()
+		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
+	}
+
+	let targetLanguage: string
+
+	if (options.targetLanguage) {
+		const languageData = await parseLangIdentifier(options.targetLanguage)
+
+		targetLanguage = languageData.Name
+
+		logger.end()
+		logger.logTitledMessage('Target language specified', formatLanguageCodeWithName(targetLanguage))
+	} else {
+		logger.start('No target language specified. Detect target language')
+		const { detectedLanguage } = await API.detectTextLanguage(translatedTranscript, options.languageDetection || {})
+
+		targetLanguage = detectedLanguage
+
+		logger.end()
+		logger.logTitledMessage('Target language detected', formatLanguageCodeWithName(detectedLanguage))
+	}
+
+	logger.start(`Load ${options.engine} module`)
+
+	let mappedWordTimeline: Timeline
+
+	switch (options.engine) {
+		case 'e5': {
+			const { alignTimelineToTextSemantically, e5SupportedLanguages } = await import('../alignment/SemanticTextAlignment.js')
+
+			const shortSourceLanguageCode = getShortLanguageCode(sourceLanguage)
+			if (!e5SupportedLanguages.includes(shortSourceLanguageCode)) {
+				throw new Error(`Source language ${formatLanguageCodeWithName(sourceLanguage)} is not supported by the E5 embedding model.`)
+			}
+
+			const shortTargetLanguageCode = getShortLanguageCode(targetLanguage)
+			if (!e5SupportedLanguages.includes(shortTargetLanguageCode)) {
+				throw new Error(`Target language ${formatLanguageCodeWithName(targetLanguage)} is not supported by the E5 embedding model.`)
+			}
+
+			logger.end()
+
+			mappedWordTimeline = await alignTimelineToTextSemantically(
+				inputTimeline,
+				translatedTranscript,
+				targetLanguage)
+
+			break
+		}
+
+		default: {
+			throw new Error(`Unsupported engine: ${options.engine}`)
+		}
+	}
+
+	logger.start(`Postprocess timeline`)
+
+	addWordTextOffsetsToTimeline(mappedWordTimeline, translatedTranscript)
+
+	const { segmentTimeline: mappedTimeline } = await wordTimelineToSegmentSentenceTimeline(mappedWordTimeline, translatedTranscript, targetLanguage)
+
+	logger.end()
+	logger.logDuration(`Total timeline translation alignment time`, startTimestamp, chalk.magentaBright)
+
+	logger.end()
+
+	return {
+		timeline: mappedTimeline,
+		wordTimeline: mappedWordTimeline,
+
+		sourceLanguage,
+		targetLanguage,
+
+		rawAudio,
+	}
+}
+
+// Types
+export interface TimelineTranslationAlignmentResult {
+	timeline: Timeline
+	wordTimeline: Timeline
+
+	sourceLanguage?: string
+	targetLanguage: string
+
+	rawAudio?: RawAudio
+}
+
+export interface TimelineTranslationAlignmentOptions {
+	engine?: 'e5'
+
+	sourceLanguage?: string
+	targetLanguage?: string
+
+	audio?: AudioSourceParam
+
+	languageDetection?: API.TextLanguageDetectionOptions
+
+	subtitles?: SubtitlesConfig
+
+	e5?: {
+		model: 'small-fp16'
+	}
+}
+
+// Constants
+const defaultTimelineTranslationAlignmentOptions: TimelineTranslationAlignmentOptions = {
+	engine: 'e5',
+
+	sourceLanguage: undefined,
+	targetLanguage: undefined,
+
+	audio: undefined,
+
+	languageDetection: undefined,
+
+	subtitles: undefined,
+
+	e5: {
+		model: 'small-fp16',
+	}
+}

package/src/api/TranscriptAndTranslationAlignment.ts

@@ -0,0 +1,164 @@
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+import { logToStderr } from '../utilities/Utilities.js'
+import { AudioSourceParam, RawAudio } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
+
+import * as API from './API.js'
+import { Timeline } from '../utilities/Timeline.js'
+import chalk from 'chalk'
+import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
+
+const log = logToStderr
+
+export async function alignTranscriptAndTranslation(input: AudioSourceParam, transcript: string, translatedTranscript: string, options: TranscriptAndTranslationAlignmentOptions): Promise<TranscriptAndTranslationAlignmentResult> {
+	const logger = new Logger()
+
+	const startTimestamp = logger.getTimestamp()
+
+	options = extendDeep(defaultTranscriptAndTranslationAlignmentOptions, options)
+
+	if (options.sourceLanguage && !options.alignment?.language) {
+		options.alignment = extendDeep(options.alignment || {}, { language: options.sourceLanguage })
+	}
+
+	if (options.targetLanguage && !options.timelineAlignment?.targetLanguage) {
+		options.timelineAlignment = extendDeep(options.timelineAlignment || {}, { targetLanguage: options.targetLanguage })
+	}
+
+	let alignmentResult: API.AlignmentResult
+	let timelineAlignmentResult: API.TimelineTranslationAlignmentResult
+
+	switch (options.engine) {
+		case 'two-stage': {
+			logger.logTitledMessage(`Start stage 1`, `Align speech to transcript`, chalk.magentaBright)
+			logger.end()
+
+			alignmentResult = await API.align(input, transcript, options.alignment || {})
+
+			logger.log(``)
+			logger.logTitledMessage(`Start stage 2`, `Align timeline to translated transcript`, chalk.magentaBright)
+			logger.end()
+
+			timelineAlignmentResult = await API.alignTimelineTranslation(alignmentResult.timeline, translatedTranscript, options.timelineAlignment || {})
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	logger.end()
+
+	logger.log(``)
+	logger.logDuration(`Total transcript and translation alignment time`, startTimestamp, chalk.magentaBright)
+
+	return {
+		timeline: alignmentResult.timeline,
+		wordTimeline: alignmentResult.wordTimeline,
+
+		translatedTimeline: timelineAlignmentResult.timeline,
+		translatedWordTimeline: timelineAlignmentResult.wordTimeline,
+
+		transcript,
+		translatedTranscript,
+
+		sourceLanguage: alignmentResult.language,
+		targetLanguage: timelineAlignmentResult.targetLanguage,
+
+		inputRawAudio: alignmentResult.inputRawAudio,
+		isolatedRawAudio: alignmentResult.isolatedRawAudio,
+		backgroundRawAudio: alignmentResult.backgroundRawAudio,
+	}
+}
+
+export interface TranscriptAndTranslationAlignmentResult {
+	timeline: Timeline
+	wordTimeline: Timeline
+
+	translatedTimeline: Timeline
+	translatedWordTimeline: Timeline
+
+	transcript: string
+	translatedTranscript: string
+
+	sourceLanguage: string
+	targetLanguage: string
+
+	inputRawAudio: RawAudio
+	isolatedRawAudio?: RawAudio
+	backgroundRawAudio?: RawAudio
+}
+
+export type TranscriptAndTranslationAlignmentEngine = 'two-stage'
+
+export interface TranscriptAndTranslationAlignmentOptions {
+	engine?: TranscriptAndTranslationAlignmentEngine
+
+	sourceLanguage?: string
+	targetLanguage?: string
+
+	isolate?: boolean
+
+	crop?: boolean
+
+	alignment?: API.AlignmentOptions
+
+	timelineAlignment?: API.TimelineTranslationAlignmentOptions
+
+	languageDetection?: API.TextLanguageDetectionOptions
+
+	vad?: API.VADOptions
+
+	plainText?: API.PlainTextOptions
+
+	subtitles?: SubtitlesConfig
+
+	sourceSeparation?: API.SourceSeparationOptions
+}
+
+export const defaultTranscriptAndTranslationAlignmentOptions: TranscriptAndTranslationAlignmentOptions = {
+	engine: 'two-stage',
+
+	sourceLanguage: undefined,
+	targetLanguage: undefined,
+
+	isolate: false,
+
+	crop: true,
+
+	alignment: {
+	},
+
+	timelineAlignment: {
+	},
+
+	languageDetection: {
+	},
+
+	plainText: {
+		paragraphBreaks: 'double',
+		whitespace: 'collapse'
+	},
+
+	subtitles: {
+	},
+
+	vad: {
+		engine: 'adaptive-gate'
+	},
+
+	sourceSeparation: {
+	},
+}
+
+export const TranscriptAndTranslationAlignmentEngines: API.EngineMetadata[] = [
+	{
+		id: 'two-stage',
+		name: 'Two-stage translation alignment',
+		description: 'Applies two-stage translation alignment to the spoken audio. First stage aligns the speech to the native language transcript. Second stage aligns the resulting timeline with the translated transcript.',
+		type: 'local'
+	}
+]