echogarden 1.2.0 → 1.3.0

package/src/api/Alignment.ts

@@ -6,7 +6,7 @@ import { Logger } from '../utilities/Logger.js'
 
 import * as API from './API.js'
 import { Timeline, addTimeOffsetToTimeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
-import { formatLanguageCodeWithName, getDefaultDialectForLanguageCodeIfPossible, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
+import { formatLanguageCodeWithName, getDefaultDialectForLanguageCodeIfPossible, getShortLanguageCode, parseLangIdentifier } from '../utilities/Locale.js'
 import { type WhisperAlignmentOptions } from '../recognition/WhisperSTT.js'
 import chalk from 'chalk'
 import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
@@ -72,14 +72,20 @@ export async function align(input: AudioSourceParam, transcript: string, options
 	let language: string
 
 	if (options.language) {
-		language = normalizeLanguageCode(options.language!)
+		const languageData = await parseLangIdentifier(options.language)
+
+		language = languageData.Name
+
+		logger.logTitledMessage('Language specified', formatLanguageCodeWithName(language))
 	} else {
 		logger.start('No language specified. Detecting language')
 		const { detectedLanguage } = await API.detectTextLanguage(transcript, options.languageDetection || {})
+
 		language = detectedLanguage
 
 		logger.end()
-		logger.logTitledMessage('Language detected', formatLanguageCodeWithName(detectedLanguage))
+
+		logger.logTitledMessage('Language detected', formatLanguageCodeWithName(language))
 	}
 
 	language = getDefaultDialectForLanguageCodeIfPossible(language)

package/src/api/Recognition.ts

@@ -6,7 +6,7 @@ import { Logger } from '../utilities/Logger.js'
 
 import * as API from './API.js'
 import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
-import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
+import { formatLanguageCodeWithName, parseLangIdentifier } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import chalk from 'chalk'
 
@@ -62,24 +62,27 @@ export async function recognize(input: AudioSourceParam, options: RecognitionOpt
 
 	const engine = options.engine!
 
-	if (!options.language) { // && options.engine != 'whisper') {
-		logger.start('No language specified. Detecting speech language')
-		const { detectedLanguage } = await API.detectSpeechLanguage(sourceRawAudio, options.languageDetection!)
+	if (options.language) {
+		const languageData = await parseLangIdentifier(options.language)
 
-		logger.end()
-		logger.logTitledMessage('Language detected', formatLanguageCodeWithName(detectedLanguage))
+		options.language = languageData.Name
 
-		options.language = detectedLanguage
-	} else {
 		logger.end()
+		logger.logTitledMessage('Language specified', formatLanguageCodeWithName(options.language))
+	} else {
+		logger.start('No language specified. Detecting speech language')
+		const { detectedLanguage } = await API.detectSpeechLanguage(sourceRawAudio, options.languageDetection!)
 
-		const specifiedLanguageFormatted = formatLanguageCodeWithName(getShortLanguageCode(normalizeLanguageCode(options.language)))
+		options.language = detectedLanguage
 
-		logger.logTitledMessage('Language specified', specifiedLanguageFormatted)
+		logger.end()
+		logger.logTitledMessage('Language detected', formatLanguageCodeWithName(detectedLanguage))
 	}
 
-	const languageCode = normalizeLanguageCode(options.language)
-	const shortLanguageCode = getShortLanguageCode(languageCode)
+	const languageData = await parseLangIdentifier(options.language)
+
+	const languageCode = languageData.Name
+	const shortLanguageCode = languageData.TwoLetterISOLanguageName
 
 	let transcript: string
 	let timeline: Timeline | undefined

package/src/api/Synthesis.ts

@@ -15,7 +15,7 @@ import { loadLexiconsForLanguage } from '../nlp/Lexicon.js'
 import * as API from './API.js'
 import { Timeline, TimelineEntry, addTimeOffsetToTimeline, multiplyTimelineByFactor } from '../utilities/Timeline.js'
 import { getAppDataDir, ensureDir, existsSync, isFileIsUpToDate, readAndParseJsonFile, writeFileSafe } from '../utilities/FileSystem.js'
-import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode, defaultDialectForLanguageCode } from '../utilities/Locale.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode, defaultDialectForLanguageCode, parseLangIdentifier, normalizeIdentifierToLangaugeCode } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import { EngineMetadata, appName } from './Common.js'
 import { shouldCancelCurrentTask } from '../server/Worker.js'
@@ -328,7 +328,15 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 	logger.start(`Initialize ${engine} module`)
 
 	const voice = selectedVoice.name
-	const language = options.language ? normalizeLanguageCode(options.language) : selectedVoice.languages[0]
+
+	let language: string
+
+	if (options.language) {
+		language = await normalizeIdentifierToLangaugeCode(options.language)
+	} else {
+		language = selectedVoice.languages[0]
+	}
+
 	const voiceGender = selectedVoice.gender
 
 	const speed = clip(options.speed!, 0.1, 10.0)
@@ -1548,7 +1556,7 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 		voiceList = await loadVoiceList()
 	}
 
-	const languageCode = normalizeLanguageCode(options.language || '')
+	const languageCode = await normalizeIdentifierToLangaugeCode(options.language || '')
 
 	if (languageCode) {
 		let filteredVoiceList = voiceList.filter(voice => voice.languages.includes(languageCode))
@@ -1611,7 +1619,7 @@ export interface RequestVoiceListResult {
 }
 
 export async function selectBestOfflineEngineForLanguage(language: string): Promise<SynthesisEngine> {
-	language = normalizeLanguageCode(language)
+	language = await normalizeIdentifierToLangaugeCode(language)
 
 	const VitsTTS = await import('../synthesis/VitsTTS.js')
 
@@ -1621,22 +1629,6 @@ export async function selectBestOfflineEngineForLanguage(language: string): Prom
 		return 'vits'
 	}
 
-	const FliteTTS = await import('../synthesis/FliteTTS.js')
-
-	const fliteLanguages = getAllLangCodesFromVoiceList(FliteTTS.voiceList)
-
-	if (fliteLanguages.includes(language)) {
-		return 'flite'
-	}
-
-	const SvoxPicoTTS = await import('../synthesis/SvoxPicoTTS.js')
-
-	const picoLanguages = getAllLangCodesFromVoiceList(SvoxPicoTTS.voiceList)
-
-	if (picoLanguages.includes(language)) {
-		return 'pico'
-	}
-
 	return 'espeak'
 }
 

package/src/api/Translation.ts

@@ -6,7 +6,7 @@ import { Logger } from '../utilities/Logger.js'
 
 import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
 import { type WhisperOptions } from '../recognition/WhisperSTT.js'
-import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeIdentifierToLangaugeCode, parseLangIdentifier } from '../utilities/Locale.js'
 import { EngineMetadata } from './Common.js'
 import { type SpeechLanguageDetectionOptions, detectSpeechLanguage } from './API.js'
 import chalk from 'chalk'
@@ -62,28 +62,31 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
 
-	if (!options.sourceLanguage) {
-		logger.start('No source language specified. Detecting speech language')
-		const { detectedLanguage } = await detectSpeechLanguage(sourceRawAudio, options.languageDetection || {})
+	if (options.sourceLanguage) {
+		const languageData = await parseLangIdentifier(options.sourceLanguage)
 
-		logger.end()
-		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
+		options.sourceLanguage = languageData.Name
 
-		options.sourceLanguage = detectedLanguage
-	} else {
 		logger.end()
+		logger.logTitledMessage('Source language specified', formatLanguageCodeWithName(options.sourceLanguage))
+	} else {
+		logger.start('No source language specified. Detecting speech language')
+		const { detectedLanguage } = await detectSpeechLanguage(sourceRawAudio, options.languageDetection || {})
 
-		const specifiedLanguageFormatted = formatLanguageCodeWithName(getShortLanguageCode(normalizeLanguageCode(options.sourceLanguage)))
+		options.sourceLanguage = detectedLanguage
 
-		logger.logTitledMessage('Source language', specifiedLanguageFormatted)
+		logger.end()
+		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
 	}
 
-	logger.logTitledMessage('Target language', formatLanguageCodeWithName(getShortLanguageCode(normalizeLanguageCode(options.targetLanguage!))))
+	options.targetLanguage = await normalizeIdentifierToLangaugeCode(options.targetLanguage!)
+
+	logger.logTitledMessage('Target language', formatLanguageCodeWithName(options.targetLanguage))
 
 	logger.start('Preprocess audio for translation')
 
 	const engine = options.engine!
-	const sourceLanguage = normalizeLanguageCode(options.sourceLanguage!)
+	const sourceLanguage = options.sourceLanguage!
 	const targetLanguage = options.targetLanguage!
 
 	let transcript: string
@@ -119,10 +122,6 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
 
 			({ transcript, timeline: wordTimeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, 'translate', sourceLanguage, whisperOptions))
 
-			addWordTextOffsetsToTimeline(wordTimeline, transcript);
-
-			({ segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(wordTimeline, transcript, targetLanguage, 'single', 'preserve'))
-
 			break
 		}
 
@@ -155,9 +154,7 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
 				modelName,
 				modelPath,
 				whisperCppOptions,
-			));
-
-			({ segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(wordTimeline, transcript, targetLanguage, 'single', 'preserve'))
+			))
 
 			break
 		}
@@ -194,13 +191,21 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
 
 	// If the audio was cropped before recognition, map the timestamps back to the original audio
 	if (sourceUncropTimeline && sourceUncropTimeline.length > 0) {
-		API.convertCroppedToUncroppedTimeline(segmentTimeline, sourceUncropTimeline)
-
 		if (wordTimeline) {
 			API.convertCroppedToUncroppedTimeline(wordTimeline, sourceUncropTimeline)
+		} else if (segmentTimeline) {
+			API.convertCroppedToUncroppedTimeline(segmentTimeline, sourceUncropTimeline)
 		}
 	}
 
+	if (wordTimeline) {
+		addWordTextOffsetsToTimeline(wordTimeline, transcript)
+	}
+
+	if (!segmentTimeline) {
+		({ segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(wordTimeline!, transcript, targetLanguage, 'single', 'preserve'))
+	}
+
 	logger.log('')
 	logger.logDuration(`Total speech translation time`, startTimestamp, chalk.magentaBright)
 

package/src/api/TranslationAlignment.ts

@@ -6,7 +6,7 @@ import { Logger } from '../utilities/Logger.js'
 
 import * as API from './API.js'
 import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
-import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeIdentifierToLangaugeCode, parseLangIdentifier } from '../utilities/Locale.js'
 import { type WhisperAlignmentOptions } from '../recognition/WhisperSTT.js'
 import chalk from 'chalk'
 import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
@@ -57,18 +57,25 @@ export async function alignTranslation(input: AudioSourceParam, transcript: stri
 	let sourceLanguage: string
 
 	if (options.sourceLanguage) {
-		sourceLanguage = normalizeLanguageCode(options.sourceLanguage!)
+		const languageData = await parseLangIdentifier(options.sourceLanguage)
+
+		sourceLanguage = languageData.Name
+
+		logger.end()
+		logger.logTitledMessage('Source language specified', formatLanguageCodeWithName(sourceLanguage))
 	} else {
 		logger.start('No source language specified. Detecting speech language')
 		const { detectedLanguage } = await API.detectSpeechLanguage(sourceRawAudio, options.languageDetection || {})
 
+		sourceLanguage = detectedLanguage
+
 		logger.end()
 		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
-
-		sourceLanguage = detectedLanguage
 	}
 
-	const targetLanguage = normalizeLanguageCode(options.targetLanguage!)
+	const targetLanguage = await normalizeIdentifierToLangaugeCode(options.targetLanguage!)
+
+	logger.logTitledMessage('Target language', formatLanguageCodeWithName(targetLanguage))
 
 	let mappedTimeline: Timeline
 

package/src/api/VoiceActivityDetection.ts

@@ -1,6 +1,6 @@
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 
-import { logToStderr } from '../utilities/Utilities.js'
+import { logToStderr, roundToDigits } from '../utilities/Utilities.js'
 import { AudioSourceParam, RawAudio, cropToTimeline, ensureRawAudio, } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 
@@ -191,35 +191,69 @@ function frameProbabilitiesToTimeline(frameProbabilities: number[], frameDuratio
 }
 
 export function convertCroppedToUncroppedTimeline(timeline: Timeline, uncropTimeline: Timeline) {
+	if (timeline.length === 0) {
+		return
+	}
+
 	for (const entry of timeline) {
-		entry.startTime = mapTimestampUsingUncropTimeline(entry.startTime, uncropTimeline)
-		entry.endTime = mapTimestampUsingUncropTimeline(entry.endTime, uncropTimeline)
+		const {
+			mappedStartTime,
+			mappedEndTime
+		} = mapUsingUncropTimeline(entry.startTime, entry.endTime, uncropTimeline)
+
+		const mapSubTimeline = (subTimeline: Timeline | undefined) => {
+			if (!subTimeline) {
+				return
+			}
+
+			for (const subEntry of subTimeline) {
+				subEntry.startTime = Math.min(mappedStartTime + (subEntry.startTime - entry.startTime), mappedEndTime)
+				subEntry.endTime = Math.min(mappedStartTime + (subEntry.endTime - entry.startTime), mappedEndTime)
 
-		if (entry.timeline) {
-			convertCroppedToUncroppedTimeline(entry.timeline, uncropTimeline)
+				mapSubTimeline(subEntry.timeline)
+			}
 		}
+
+		mapSubTimeline(entry.timeline)
+
+		entry.startTime = mappedStartTime
+		entry.endTime = mappedEndTime
 	}
 }
 
-export function mapTimestampUsingUncropTimeline(timeInCroppedAudio: number, uncropTimeline: Timeline) {
+function mapUsingUncropTimeline(startTimeInCroppedAudio: number, endTimeInCroppedAudio: number, uncropTimeline: Timeline) {
 	let offsetInCroppedAudio = 0
 
-	for (let i = 0; i < uncropTimeline.length; i++) {
-		const entry = uncropTimeline[i]
+	let bestOverlapDuration = -1
+	let mappedStartTime = -1
+	let mappedEndTime = -1
 
-		const entryDuration = entry.endTime - entry.startTime
+	for (const uncropEntry of uncropTimeline) {
+		const uncropEntryDuration = uncropEntry.endTime - uncropEntry.startTime
 
-		const endOffset = offsetInCroppedAudio + entryDuration
+		const overlapStartTime = Math.max(startTimeInCroppedAudio, offsetInCroppedAudio)
+		const overlapEndTime = Math.min(endTimeInCroppedAudio, offsetInCroppedAudio + uncropEntryDuration)
 
-		if ((i === uncropTimeline.length - 1) ||
-			(timeInCroppedAudio >= offsetInCroppedAudio && timeInCroppedAudio < endOffset)) {
-			return entry.startTime + (timeInCroppedAudio - offsetInCroppedAudio)
+		const overlapDuration = overlapEndTime - overlapStartTime
+
+		if (overlapDuration >= 0 && overlapDuration > bestOverlapDuration) {
+			bestOverlapDuration = overlapDuration
+
+			mappedStartTime = uncropEntry.startTime + (overlapStartTime - offsetInCroppedAudio)
+			mappedEndTime = uncropEntry.startTime + (overlapEndTime - offsetInCroppedAudio)
 		}
 
-		offsetInCroppedAudio += entryDuration
+		offsetInCroppedAudio += uncropEntryDuration
+	}
+
+	if (bestOverlapDuration === -1) {
+		throw new Error(`No match found in uncrop timeline (should not occur)`)
 	}
 
-	throw new Error(`Should not be reached`)
+	return {
+		mappedStartTime,
+		mappedEndTime
+	}
 }
 
 export interface VADResult {

package/src/cli/CLI.ts

@@ -102,7 +102,11 @@ export async function start(processArgs: string[]) {
 			process.exit(1)
 		}
 
-		const { operationArgs, parsedArgumentsLookup }  = parseCLIArguments(processArgs.slice(1))
+		const { operationArgs, parsedArgumentsLookup } = parseCLIArguments(processArgs.slice(1))
+
+		const globalOptionsLookup = new Map<string, string>()
+		const cliOptionsLookup = new Map<string, string>()
+		const operationsOptionsLookup = new Map<string, string>()
 
 		if (!parsedArgumentsLookup.has('config')) {
 			const defaultConfigFile = `./${appName}.config`
@@ -135,10 +139,6 @@ export async function start(processArgs: string[]) {
 				sectionName = 'speak'
 			}
 
-			const globalOptionsLookup = new Map<string, string>()
-			const cliOptionsLookup = new Map<string, string>()
-			const operationsOptionsLookup = new Map<string, string>()
-
 			if (parsedConfigFile.has('global')) {
 				for (const [key, value] of parsedConfigFile.get('global')!) {
 					globalOptionsLookup.set(key, value)
@@ -156,27 +156,27 @@ export async function start(processArgs: string[]) {
 					operationsOptionsLookup.set(key, value)
 				}
 			}
+		}
 
-			const globalOptionsKeys = API.listGlobalOptions()
-			const cliOptionsKeys = CLIOptionsKeys
+		const globalOptionsKeys = API.listGlobalOptions()
+		const cliOptionsKeys = CLIOptionsKeys
 
-			for (const [key, value] of parsedArgumentsLookup) {
-				if (globalOptionsKeys.includes(key)) {
-					globalOptionsLookup.set(key, value)
-				} else if (cliOptionsKeys.includes(key as any)) {
-					cliOptionsLookup.set(key, value)
-				} else {
-					operationsOptionsLookup.set(key, value)
-				}
+		for (const [key, value] of parsedArgumentsLookup) {
+			if (globalOptionsKeys.includes(key)) {
+				globalOptionsLookup.set(key, value)
+			} else if (cliOptionsKeys.includes(key as any)) {
+				cliOptionsLookup.set(key, value)
+			} else {
+				operationsOptionsLookup.set(key, value)
 			}
+		}
 
-			operationData.operation = operation
-			operationData.operationArgs = operationArgs
+		operationData.operation = operation
+		operationData.operationArgs = operationArgs
 
-			operationData.globalOptions = await optionsLookupToTypedObject(globalOptionsLookup, 'GlobalOptions')
-			operationData.cliOptions = await optionsLookupToTypedObject(cliOptionsLookup, 'CLIOptions')
-			operationData.operationOptionsLookup = operationsOptionsLookup
-		}
+		operationData.globalOptions = await optionsLookupToTypedObject(globalOptionsLookup, 'GlobalOptions')
+		operationData.cliOptions = await optionsLookupToTypedObject(cliOptionsLookup, 'CLIOptions')
+		operationData.operationOptionsLookup = operationsOptionsLookup
 	} catch (e: any) {
 		resetActiveLogger()
 
@@ -997,7 +997,7 @@ async function detectVoiceActivity(operationData: CLIOperationData) {
 		const normalizedAudio = normalizeAudioLevel(inputRawAudio)
 
 		const timelineToPlay = verboseTimeline.map(entry => {
-			return {...entry, type: 'word' } as TimelineEntry
+			return { ...entry, type: 'word' } as TimelineEntry
 		})
 
 		await playAudioWithWordTimeline(normalizedAudio, timelineToPlay)

package/src/recognition/WhisperSTT.ts

@@ -13,7 +13,7 @@ import { getRawAudioDuration, RawAudio, sliceRawAudio } from '../audio/AudioUtil
 import { readFile } from '../utilities/FileSystem.js'
 import path from 'path'
 import type { LanguageDetectionResults } from '../api/API.js'
-import { getShortLanguageCode, languageCodeToName } from '../utilities/Locale.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, languageCodeToName } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import chalk from 'chalk'
 import { XorShift32RNG } from '../utilities/RandomGenerator.js'
@@ -35,13 +35,13 @@ export async function recognize(
 	options = extendDeep(defaultWhisperOptions, options)
 
 	if (sourceRawAudio.sampleRate != 16000) {
-		throw new Error('Source audio must have a sampling rate of 16000')
+		throw new Error('Source audio must have a sampling rate of 16000 Hz')
 	}
 
 	sourceLanguage = getShortLanguageCode(sourceLanguage)
 
 	if (!(sourceLanguage in languageIdLookup)) {
-		throw new Error(`The language ${languageCodeToName(sourceLanguage)} is not supported by the Whisper engine.`)
+		throw new Error(`The language ${formatLanguageCodeWithName(sourceLanguage)} is not supported by the Whisper engine.`)
 	}
 
 	if (isEnglishOnlyModel(modelName) && sourceLanguage != 'en') {
@@ -93,7 +93,7 @@ export async function align(
 	sourceLanguage = getShortLanguageCode(sourceLanguage)
 
 	if (!(sourceLanguage in languageIdLookup)) {
-		throw new Error(`The language ${languageCodeToName(sourceLanguage)} is not supported by the Whisper engine.`)
+		throw new Error(`The language ${formatLanguageCodeWithName(sourceLanguage)} is not supported by the Whisper engine.`)
 	}
 
 	if (isEnglishOnlyModel(modelName) && sourceLanguage != 'en') {
@@ -134,7 +134,7 @@ export async function alignEnglishTranslation(
 	sourceLanguage = getShortLanguageCode(sourceLanguage)
 
 	if (!(sourceLanguage in languageIdLookup)) {
-		throw new Error(`The source language ${languageCodeToName(sourceLanguage)} is not supported by the Whisper engine.`)
+		throw new Error(`The source language ${formatLanguageCodeWithName(sourceLanguage)} is not supported by the Whisper engine.`)
 	}
 
 	if (isEnglishOnlyModel(modelName)) {
@@ -784,6 +784,8 @@ export class Whisper {
 					return false
 				}
 
+				// If this is the first token in the part, unconditionally decode a timestamp token
+				// for time 0.0
 				if (isInitialState) {
 					addToken(timestampTokensStart, timestampTokenLogits, 1.0, crossAttentionQKsForToken)
 

package/src/utilities/Locale.ts

@@ -22,6 +22,39 @@ export function formatLanguageCodeWithName(languageCode: string, styleId: 1 | 2
 	}
 }
 
+export async function normalizeIdentifierToLangaugeCode(langIdentifier: string) {
+	const result = await parseLangIdentifier(langIdentifier)
+
+	return result.Name
+}
+
+export async function normalizeIdentifierToShortLanguageCode(langIdentifier: string) {
+	const result = await parseLangIdentifier(langIdentifier)
+
+	return result.TwoLetterISOLanguageName
+}
+
+export async function parseLangIdentifier(langIdentifier: string) {
+	if (!langIdentifier) {
+		return emptyLangInfoEntry
+	}
+
+	await loadLangInfoEntriesIfNeeded()
+
+	langIdentifier = langIdentifier.trim().toLowerCase()
+
+	for (const entry of langInfoEntries) {
+		if (langIdentifier === entry.NameLowerCase ||
+			langIdentifier === entry.ThreeLetterISOLanguageName ||
+			langIdentifier === entry.EnglishNameLowerCase) {
+
+			return entry
+		}
+	}
+
+	throw new Error(`Couldn't parse language identifier '${langIdentifier}'.`)
+}
+
 export function getShortLanguageCode(langCode: string) {
 	const dashIndex = langCode.indexOf('-')
 
@@ -48,7 +81,7 @@ export function normalizeLanguageCode(langCode: string) {
 
 const isoToLcidLookup = new Map<string, number>()
 const lcidToIsoLookup = new Map<number, string[]>()
-const lcidEntries: LCIDEntry[] = []
+let langInfoEntries: LangInfoEntry[] = []
 
 export async function isoToLcidLanguageCode(iso: string) {
 	await loadLcidLookupIfNeeded()
@@ -63,19 +96,13 @@ export async function lcidToIsoLanguageCode(lcid: number) {
 }
 
 async function loadLcidLookupIfNeeded() {
-	if (lcidEntries.length > 0) {
-		return lcidEntries
-	}
-
-	const lcidLookup: LCIDLookup = await readAndParseJsonFile(resolveToModuleRootDir('data/tables/lcid-table.json'))
-
-	for (const isoName in lcidLookup) {
-		const lcidEntry = lcidLookup[isoName]
-		lcidEntries.push(lcidEntry)
+	await loadLangInfoEntriesIfNeeded()
 
+	for (const lcidEntry of langInfoEntries) {
+		const name = lcidEntry.Name
 		const lcidValue = lcidEntry.LCID
 
-		isoToLcidLookup.set(isoName, lcidValue)
+		isoToLcidLookup.set(name, lcidValue)
 
 		let entry = lcidToIsoLookup.get(lcidValue)
 
@@ -84,10 +111,25 @@ async function loadLcidLookupIfNeeded() {
 			lcidToIsoLookup.set(lcidValue, entry)
 		}
 
-		entry.push(isoName)
+		entry.push(name)
 	}
 
-	return lcidEntries
+	return langInfoEntries
+}
+
+async function loadLangInfoEntriesIfNeeded() {
+	if (langInfoEntries.length > 0) {
+		return
+	}
+
+	const entries = await readAndParseJsonFile(resolveToModuleRootDir('data/tables/lcid-table.json')) as LangInfoEntry[]
+
+	for (const entry of entries) {
+		entry.NameLowerCase = entry.Name.toLowerCase()
+		entry.EnglishNameLowerCase = entry.EnglishName.toLowerCase()
+
+		langInfoEntries.push(entry)
+	}
 }
 
 export function getDefaultDialectForLanguageCodeIfPossible(langCode: string) {
@@ -107,14 +149,34 @@ export const defaultDialectForLanguageCode: { [lang: string]: string } = {
 	'nl': 'nl-NL'
 }
 
-type LCIDLookup = { [isoLangCode: string]: LCIDEntry }
+export interface LangInfoEntry {
+	LCID: number
+
+	Name: string
+	NameLowerCase: string
+
+	TwoLetterISOLanguageName: string
+	ThreeLetterISOLanguageName: string
+	ThreeLetterWindowsLanguageName: string
+
+	EnglishName: string
+	EnglishNameLowerCase: string
+
+	ANSICodePage: string
+}
+
+export const emptyLangInfoEntry: LangInfoEntry = {
+	LCID: -1,
+
+	Name: '',
+	NameLowerCase: '',
+
+	TwoLetterISOLanguageName: '',
+	ThreeLetterISOLanguageName: '',
+	ThreeLetterWindowsLanguageName: '',
+
+	EnglishName: 'Empty',
+	EnglishNameLowerCase: 'empty',
 
-export interface LCIDEntry {
-	'LCID': number
-	'Name': string
-	'TwoLetterISOLanguageName': string,
-	'ThreeLetterISOLanguageName': string,
-	'ThreeLetterWindowsLanguageName': string,
-	'EnglishName': string
-	'ANSICodePage': string
+	ANSICodePage: ''
 }