echogarden 1.5.0 → 1.6.0

package/src/alignment/DTWMfccSequenceAlignment.ts

@@ -1,15 +1,15 @@
-import { cosineDistancePrecomputedMagnitudes, createVectorForIntegerRange, euclidianDistance, euclidianDistance13Dim, magnitude } from '../math/VectorMath.js'
+import { cosineDistancePrecomputedMagnitudes, createVectorForIntegerRange, euclideanDistance, euclideanDistance13Dim, magnitude } from '../math/VectorMath.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { alignDTWWindowed } from './DTWSequenceAlignmentWindowed.js'
 
 const log = logToStderr
 
-export async function alignMFCC_DTW(mfccFrames1: number[][], mfccFrames2: number[][], windowLength: number, distanceFunctionKind: 'euclidian' | 'cosine' = 'euclidian', centerIndexes?: number[]) {
-	if (distanceFunctionKind == 'euclidian') {
-		let distanceFunction = euclidianDistance
+export async function alignMFCC_DTW(mfccFrames1: number[][], mfccFrames2: number[][], windowLength: number, distanceFunctionKind: 'euclidean' | 'cosine' = 'euclidean', centerIndexes?: number[]) {
+	if (distanceFunctionKind == 'euclidean') {
+		let distanceFunction = euclideanDistance
 
 		if (mfccFrames1.length > 0 && mfccFrames1[0].length === 13) {
-			distanceFunction = euclidianDistance13Dim
+			distanceFunction = euclideanDistance13Dim
 		}
 
 		const { path } = alignDTWWindowed(

package/src/alignment/DTWSequenceAlignmentWindowed.ts

@@ -4,9 +4,7 @@ import { AlignmentPath } from './SpeechAlignment.js'
 const log = logToStderr
 
 export function alignDTWWindowed<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, windowMaxLength: number, centerIndexes?: number[]) {
-	if (windowMaxLength < 2) {
-		throw new Error('Window length must be greater or equal to 2')
-	}
+	windowMaxLength = Math.max(windowMaxLength, 2)
 
 	if (sequence1.length == 0 || sequence2.length == 0) {
 		return {

package/src/alignment/SemanticTextAlignment.ts

@@ -118,7 +118,7 @@ export async function alignTimelineToTextSemantically(timeline: Timeline, text:
 	return resultTimeline
 }
 
-export async function alignWordsToWordsSemantically(wordsGroups1: string[][], wordsGroups2: string[][], windowTokenCount = 1000 * 1000) {
+export async function alignWordsToWordsSemantically(wordsGroups1: string[][], wordsGroups2: string[][], windowTokenCount = 20000) {
 	const logger = new Logger()
 
 	// Load embedding model

package/src/alignment/SpeechAlignment.ts

@@ -5,14 +5,14 @@ import * as API from '../api/API.js'
 import { computeMFCCs, extendDefaultMfccOptions, MfccOptions } from '../dsp/MFCC.js'
 import { alignMFCC_DTW, getCostMatrixMemorySizeMB } from './DTWMfccSequenceAlignment.js'
 import { Logger } from '../utilities/Logger.js'
-import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
-import { downmixToMonoAndNormalize, getEndingSilentSampleCount, getRawAudioDuration, getStartingSilentSampleCount, RawAudio } from '../audio/AudioUtilities.js'
+import { addTimeOffsetToTimeline, Timeline, TimelineEntry } from '../utilities/Timeline.js'
+import { concatAudioSegments, downmixToMonoAndNormalize, getEmptyRawAudio, getEndingSilentSampleCount, getRawAudioDuration, getStartingSilentSampleCount, RawAudio } from '../audio/AudioUtilities.js'
 import chalk from 'chalk'
 import { synthesize } from '../api/API.js'
 import { resampleAudioSpeex } from '../dsp/SpeexResampler.js'
 import { deepClone } from '../utilities/ObjectUtilities.js'
-import { cosineDistance, euclidianDistance, zeroIfNaN } from '../math/VectorMath.js'
-import { EspeakOptions } from '../synthesis/EspeakTTS.js'
+import { cosineDistance, euclideanDistance, zeroIfNaN } from '../math/VectorMath.js'
+import { EspeakEvent, EspeakOptions } from '../synthesis/EspeakTTS.js'
 import { alignDTWWindowed } from './DTWSequenceAlignmentWindowed.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import path from 'path'
@@ -278,7 +278,7 @@ export async function alignUsingDtwWithRecognition(
 	return result
 }
 
-// This is experimental code. It does't work well enough to be usable for anything.
+// This is experimental code. It doesn't work well enough to be usable for anything.
 // Just testing some alternative approaches.
 export async function alignUsingDtwWithEmbeddings(
 	sourceRawAudio: RawAudio,
@@ -580,7 +580,38 @@ export async function createAlignmentReferenceUsingEspeakForFragments(fragments:
 
 	progressLogger.start("Synthesize alignment reference with eSpeak")
 
-	const result = await Espeak.synthesizeFragments(fragments, espeakOptions)
+	const result = {
+		rawAudio: getEmptyRawAudio(1, await Espeak.getSampleRate()) as RawAudio,
+		timeline: [] as Timeline,
+		events: [] as EspeakEvent[],
+	}
+
+	// Split fragments to chunks and process each chunk individually,
+	// and incrementally merge the chunks to the final result.
+	{
+		const maxFragmentsInChunk = 1000
+
+		let timeOffset = 0
+
+		for (let startOffset = 0; startOffset < fragments.length; startOffset += maxFragmentsInChunk) {
+			const chunk = fragments.slice(startOffset, startOffset + maxFragmentsInChunk)
+
+			const chunkResult = await Espeak.synthesizeFragments(chunk, espeakOptions)
+
+			result.rawAudio = {
+				sampleRate: result.rawAudio.sampleRate,
+				audioChannels: concatAudioSegments([result.rawAudio.audioChannels, chunkResult.rawAudio.audioChannels])
+			}
+
+			const chunkTimeline = addTimeOffsetToTimeline(chunkResult.timeline, timeOffset)
+
+			result.timeline = [...result.timeline, ...chunkTimeline]
+
+			result.events = [...result.events, ...chunkResult.events]
+
+			timeOffset += getRawAudioDuration(chunkResult.rawAudio)
+		}
+	}
 
 	result.timeline = result.timeline.flatMap(clause => clause.timeline!)
 

package/src/api/TextTranslation.ts

@@ -1,9 +1,177 @@
-export async function translateText(input: string, options: TextTranslationOptions): Promise<TextTranslationResult> {
-	return {}
+import chalk from 'chalk'
+import { formatLanguageCodeWithName, normalizeIdentifierToLanguageCode, parseLangIdentifier } from '../utilities/Locale.js'
+import { Logger } from '../utilities/Logger.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import * as API from './API.js'
+
+export async function translateText(inputText: string, options: TextTranslationOptions): Promise<TextTranslationResult> {
+	const logger = new Logger()
+
+	const startTimestamp = logger.getTimestamp()
+
+	options = extendDeep(defaultTextTranslationOptions, options)
+
+	if (options.sourceLanguage) {
+		const languageData = await parseLangIdentifier(options.sourceLanguage)
+
+		options.sourceLanguage = languageData.Name
+
+		logger.end()
+		logger.logTitledMessage('Source language specified', formatLanguageCodeWithName(options.sourceLanguage))
+	} else {
+		logger.start('No source language specified. Detect text language')
+		const { detectedLanguage } = await API.detectTextLanguage(inputText, options.languageDetection || {})
+
+		options.sourceLanguage = detectedLanguage
+
+		logger.end()
+		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
+	}
+
+	options.targetLanguage = await normalizeIdentifierToLanguageCode(options.targetLanguage!)
+
+	logger.logTitledMessage('Target language', formatLanguageCodeWithName(options.targetLanguage))
+
+	logger.start(`Load ${options.engine} module`)
+
+	let translationPairs: TranslationPair[]
+
+	switch (options.engine) {
+		case 'nllb': {
+			const NLLBTextTranslation = await import('../text-translation/NLLBTextTranslation.js')
+
+			logger.end()
+
+			logger.logTitledMessage(`Warning`, `The nllb engine is currently an early prototype implementation and doesn't work correctly.`, chalk.yellow, 'warning')
+
+			translationPairs = await NLLBTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage)
+
+			break
+		}
+
+		case 'google-translate': {
+			const GoogleTranslateTextTranslation = await import('../text-translation/GoogleTranslateTextTranslation.js')
+
+			logger.end()
+
+			translationPairs = await GoogleTranslateTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage)
+
+			break
+		}
+
+		case 'deepl': {
+			const DeepLTextTranslation = await import('../text-translation/DeepLTextTranslation.js')
+
+			logger.end()
+
+			logger.logTitledMessage(`Warning`, `The deepl engine is currently an early prototype implementation and doesn't work correctly.`, chalk.yellow, 'warning')
+
+			translationPairs = await DeepLTextTranslation.translateText(inputText, options.sourceLanguage, options.targetLanguage)
+
+			break
+		}
+
+		default: {
+			throw new Error(`'${options.engine}' is not a supported text translation engine.`)
+		}
+	}
+
+	const translatedText = translationPairs.map(pair => {
+		const translated = pair.translatedText
+
+		if (translated.endsWith(' ') || translated.endsWith('\n')) {
+			return pair.translatedText
+		} else {
+			return pair.translatedText + ' '
+		}
+	}).join('').trim()
+
+	logger.end()
+
+	logger.log('')
+	logger.logDuration(`Total text translation time`, startTimestamp, chalk.magentaBright)
+
+	return {
+		text: inputText,
+		translatedText,
+
+		translationPairs,
+
+		sourceLanguage: options.sourceLanguage!,
+		targetLanguage: options.targetLanguage!,
+	}
 }
 
 export interface TextTranslationOptions {
+	engine?: TextTranslationEngine
+
+	sourceLanguage?: string
+	targetLanguage?: string
+
+	languageDetection?: API.TextLanguageDetectionOptions
+
+	nllb?: {
+	},
+
+	googleTranslate?: {
+	},
+
+	deepl?: {
+	},
 }
 
 export interface TextTranslationResult {
+	text: string
+	translatedText: string
+
+	translationPairs: TranslationPair[]
+
+	sourceLanguage: string
+	targetLanguage: string
 }
+
+export type TextTranslationEngine = 'nllb' | 'google-translate' | 'deepl'
+
+export interface TranslationPair {
+	sourceText: string
+	translatedText: string
+}
+
+export const defaultTextTranslationOptions: TextTranslationOptions = {
+	engine: 'google-translate',
+
+	sourceLanguage: undefined,
+	targetLanguage: 'en',
+
+	languageDetection: undefined,
+
+	nllb: {
+	},
+
+	googleTranslate: {
+	},
+
+	deepl: {
+	},
+}
+
+export const textTranslationEngines: API.EngineMetadata[] = [
+	{
+		id: 'nllb',
+		name: 'NLLB',
+		description: 'No Language Left Behind (NLLB) is a deep learning machine translation model by Facebook Research (early prototype implementation).',
+		type: 'local'
+	},
+	{
+		id: 'google-translate',
+		name: 'Google Translate',
+		description: 'Unoffical text translation API used by the Google Translate web interface.',
+		type: 'cloud'
+	},
+	{
+		id: 'deepl',
+		name: 'DeepL',
+		description: 'Unoffical text translation API used by the DeepL web interface (early prototype implementation).',
+		type: 'cloud'
+	},
+]

package/src/api/TimelineTranslationAlignment.ts

@@ -60,7 +60,7 @@ export async function alignTimelineTranslation(inputTimeline: Timeline, translat
 		logger.logTitledMessage('Target language detected', formatLanguageCodeWithName(detectedLanguage))
 	}
 
-	logger.log(`Load ${options.engine} module`)
+	logger.start(`Load ${options.engine} module`)
 
 	let mappedWordTimeline: Timeline
 

package/src/audio/AudioPlayer.ts

@@ -274,6 +274,7 @@ export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosit
 	})
 }
 
+/*
 export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (timePosition: number) => void, microFadeInOut = true) {
 	return new Promise<void>(async (resolve, reject) => {
 		if (microFadeInOut) {
@@ -354,6 +355,7 @@ export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (t
 		}
 	})
 }
+*/
 
 export const charactersToWriteAhead = [
 	',', '.', '，', '、', '：', '；',

package/src/cli/CLI.ts

@@ -1,12 +1,12 @@
 import * as API from '../api/API.js'
 import { parseCLIArguments } from './CLIParser.js'
-import { clip, convertHtmlToText, formatIntegerWithLeadingZeros, formatListWithQuotedElements, getWithDefault, logToStderr, setupUnhandledExceptionListeners, splitFilenameOnExtendedExtension, stringifyAndFormatJson } from '../utilities/Utilities.js'
+import { convertHtmlToText, formatIntegerWithLeadingZeros, formatListWithQuotedElements, getWithDefault, logToStderr, setupUnhandledExceptionListeners, splitFilenameOnExtendedExtension, stringifyAndFormatJson } from '../utilities/Utilities.js'
 import { getOptionTypeFromSchema, SchemaTypeDefinition } from './CLIOptionsSchema.js'
 import { ParsedConfigFile, parseConfigFile, parseJSONConfigFile } from './CLIConfigFile.js'
 
 import chalk from 'chalk'
 import { RawAudio, applyGainDecibels, encodeRawAudioToWave, getEmptyRawAudio, getRawAudioDuration, normalizeAudioLevel, sliceRawAudioByTime } from '../audio/AudioUtilities.js'
-import { SubtitlesConfig, subtitlesToText, subtitlesToTimeline, timelineToSubtitles } from '../subtitles/Subtitles.js'
+import { SubtitlesConfig, subtitlesToText, timelineToSubtitles } from '../subtitles/Subtitles.js'
 import { Logger, resetActiveLogger } from '../utilities/Logger.js'
 import { isMainThread, parentPort } from 'node:worker_threads'
 import { encodeFromChannels, getDefaultFFMpegOptionsForSpeech } from '../codecs/FFMpegTranscoder.js'
@@ -224,18 +224,20 @@ const help = [
 	`    Transcribe a spoken audio file\n`,
 	`${executableName} ${chalk.magentaBright('align')} audioFile transcriptFile [output files...] [options...]`,
 	`    Align spoken audio file to its transcript\n`,
+	`${executableName} ${chalk.magentaBright('translate-text')} inputFile [output files...] [options...]`,
+	`    Translate text to a different language\n`,
 	`${executableName} ${chalk.magentaBright('translate-speech')} audioFile [output files...] [options...]`,
-	`    Transcribe audio file directly to a different language\n`,
+	`    Transcribe spoken audio file directly to a different language\n`,
 	`${executableName} ${chalk.magentaBright('align-translation')} audioFile translatedTranscriptFile [output files...] [options...]`,
 	`    Align spoken audio file to its translated transcript\n`,
 	`${executableName} ${chalk.magentaBright('align-transcript-and-translation')} audioFile transcriptFile translatedTranscriptFile [output files...] [options...]`,
 	`    Align spoken audio file to both its transcript and its translated transcript using a two-stage approach.\n`,
 	`${executableName} ${chalk.magentaBright('align-timeline-translation')} timelineFile translatedFile [output files...] [options...]`,
 	`    Align a given timeline file to its translated text\n`,
-	`${executableName} ${chalk.magentaBright('detect-speech-language')} audioFile [output files...] [options...]`,
-	`    Detect language of spoken audio file\n`,
 	`${executableName} ${chalk.magentaBright('detect-text-language')} inputFile [output files...] [options...]`,
 	`    Detect language of textual file\n`,
+	`${executableName} ${chalk.magentaBright('detect-speech-language')} audioFile [output files...] [options...]`,
+	`    Detect language of spoken audio file\n`,
 	`${executableName} ${chalk.magentaBright('detect-voice-activity')} audioFile [output files...] [options...]`,
 	`    Detect voice activity in audio file\n`,
 	`${executableName} ${chalk.magentaBright('denoise')} audioFile [output files...] [options...]`,
@@ -279,6 +281,11 @@ async function startWithArgs(operationData: CLIOperationData) {
 			break
 		}
 
+		case 'translate-text': {
+			await translateText(operationData)
+			break
+		}
+
 		case 'translate-speech': {
 			await translateSpeech(operationData)
 			break
@@ -1014,6 +1021,75 @@ export async function alignTimelineTranslation(operationData: CLIOperationData)
 	}
 }
 
+export async function translateText(operationData: CLIOperationData) {
+	const logger = new Logger()
+
+	const { operationArgs, operationOptionsLookup, cliOptions } = operationData
+
+	const inputFilename = operationArgs[0]
+	const outputFilenames = operationArgs.slice(1)
+
+	if (inputFilename == undefined) {
+		throw new Error(`translate-text requires an argument containing the input file path.`)
+	}
+
+	if (!existsSync(inputFilename)) {
+		throw new Error(`The given input file '${inputFilename}' was not found.`)
+	}
+
+	const inputFileExtension = getLowercaseFileExtension(inputFilename)
+	const inputFileContent = await readFile(inputFilename, { encoding: 'utf-8' })
+
+	let inputText: string
+
+	if (inputFileExtension === 'txt') {
+		inputText = inputFileContent
+	} else if (inputFileExtension === 'html' || inputFileExtension === 'htm') {
+		inputText = await convertHtmlToText(inputFileContent)
+	} else if (inputFileExtension == 'srt' || inputFileExtension == 'vtt') {
+		inputText = subtitlesToText(inputFileContent)
+	} else {
+		throw new Error(`align only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
+	}
+
+	const options = await optionsLookupToTypedObject(operationOptionsLookup, 'TextTranslationOptions')
+
+	const allowOverwrite = getWithDefault(cliOptions.overwrite, overwriteByDefault)
+
+	await checkOutputFilenames(outputFilenames, false, true, true)
+
+	const {
+		text,
+		translatedText,
+
+		translationPairs,
+
+		sourceLanguage,
+		targetLanguage,
+	} = await API.translateText(inputFileContent, options)
+
+	if (outputFilenames.length > 0) {
+		logger.start('\nWrite output files')
+
+		for (const outputFilename of outputFilenames) {
+			const partPatternMatch = outputFilename.match(filenamePlaceholderPattern)
+
+			if (partPatternMatch) {
+				continue
+			}
+
+			const fileSaver = getFileSaver(outputFilename, allowOverwrite)
+
+			await fileSaver(getEmptyRawAudio(1, 16000), translationPairs as any as Timeline, translatedText, undefined)
+		}
+
+		logger.end()
+	} else {
+		logger.log(``)
+		logger.log(translatedText)
+	}
+}
+
 export async function translateSpeech(operationData: CLIOperationData) {
 	const logger = new Logger()
 
@@ -1040,7 +1116,19 @@ export async function translateSpeech(operationData: CLIOperationData) {
 
 	await checkOutputFilenames(outputFilenames, true, true, true)
 
-	const { transcript, timeline, wordTimeline, sourceLanguage, targetLanguage, inputRawAudio, isolatedRawAudio, backgroundRawAudio } = await API.translateSpeech(inputFilename, options)
+	const {
+		transcript,
+
+		timeline,
+		wordTimeline,
+
+		sourceLanguage,
+		targetLanguage,
+
+		inputRawAudio,
+		isolatedRawAudio,
+		backgroundRawAudio
+	} = await API.translateSpeech(inputFilename, options)
 
 	if (outputFilenames.length > 0) {
 		logger.start('\nWrite output files')
@@ -1382,6 +1470,12 @@ export async function listEngines(operationData: CLIOperationData) {
 			break
 		}
 
+		case 'translate-text': {
+			engines = API.textTranslationEngines
+
+			break
+		}
+
 		case 'translate-speech': {
 			engines = API.speechTranslationEngines
 
@@ -1432,7 +1526,7 @@ export async function listEngines(operationData: CLIOperationData) {
 		}
 
 		default: {
-			throw new Error(`Unrecognized operation: '${targetOperation}'`)
+			throw new Error(`Unrecognized operation name: '${targetOperation}'`)
 		}
 	}
 

package/src/math/VectorMath.ts

@@ -456,11 +456,11 @@ export function meanSquaredError(actual: ArrayLike<number>, expected: ArrayLike<
 	return sum / featureCount
 }
 
-export function euclidianDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
-	return Math.sqrt(squaredEuclidianDistance(vector1, vector2))
+export function euclideanDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
+	return Math.sqrt(squaredEuclideanDistance(vector1, vector2))
 }
 
-export function squaredEuclidianDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
+export function squaredEuclideanDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
 	if (vector1.length !== vector2.length) {
 		throw new Error('Vectors are not the same length')
 	}
@@ -480,11 +480,11 @@ export function squaredEuclidianDistance(vector1: ArrayLike<number>, vector2: Ar
 	return sum
 }
 
-export function euclidianDistance13Dim(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
-	return Math.sqrt(squaredEuclidianDistance13Dim(vector1, vector2))
+export function euclideanDistance13Dim(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
+	return Math.sqrt(squaredEuclideanDistance13Dim(vector1, vector2))
 }
 
-export function squaredEuclidianDistance13Dim(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
+export function squaredEuclideanDistance13Dim(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
 	// Assumes the input has 13 dimensions (optimized for 13-dimensional MFCC vectors)
 
 	const result =

package/src/nlp/ChineseSegmentation.ts

@@ -1,5 +1,5 @@
 export async function splitChineseTextToWords_Jieba(text: string, fineGrained = false, useHMM = true) {
-	const jieba = await getWasmInstance()
+	const jieba = await getJiebaWasmInstance()
 
 	if (!fineGrained) {
 		return jieba.cut(text, useHMM)
@@ -58,10 +58,12 @@ export async function splitChineseTextToWords_Jieba(text: string, fineGrained =
 }
 
 let JiebaWasmInstance: typeof import('jieba-wasm')
-async function getWasmInstance() {
+
+async function getJiebaWasmInstance() {
 	if (!JiebaWasmInstance) {
-		const { default: JibeaWasm } = await import('jieba-wasm')
-		JiebaWasmInstance = JibeaWasm
+		const { default: JiebaWasm } = await import('jieba-wasm')
+
+		JiebaWasmInstance = JiebaWasm
 	}
 
 	return JiebaWasmInstance

package/src/nlp/Segmentation.ts

@@ -11,8 +11,7 @@ const log = logToStderr
 export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u
 export const punctuationPattern = /[\p{Punctuation}]/u
 
-export const phraseSeparators = [',', ';', ':']
-export const sentenceSeparators = ['.', '?', '!']
+export const phraseSeparators = [',', ';', ':', '，', '、']
 export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '=', '¼', '½', '¾']
 
 export function isWordOrSymbolWord(str: string) {
@@ -226,29 +225,35 @@ export async function splitToWords(text: string, langCode: string): Promise<stri
 	}
 }
 
-export function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakType, whitespace: WhitespaceProcessing) {
+export function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakType, whitespaceProcessingMethod: WhitespaceProcessing) {
 	let paragraphs: string[] = []
 
-	if (paragraphBreaks == 'single') {
+	if (paragraphBreaks === 'single') {
 		paragraphs = text.split(/(\r?\n)+/g)
-	} else if (paragraphBreaks == 'double') {
+	} else if (paragraphBreaks === 'double') {
 		paragraphs = text.split(/(\r?\n)(\r?\n)+/g)
 	} else {
-		throw new Error(`Invalid paragraph break type: ${paragraphBreaks}`)
+		throw new Error(`Invalid paragraph break type: '${paragraphBreaks}'`)
 	}
 
-	if (whitespace == 'removeLineBreaks') {
-		paragraphs = paragraphs.map(p => p.replaceAll(/(\r?\n)+/g, ' '))
-	} else if (whitespace == 'collapse') {
-		paragraphs = paragraphs.map(p => p.replaceAll(/\s+/g, ' '))
-	}
-
-	paragraphs = paragraphs.map(p => p.trim())
+	paragraphs = paragraphs.map(p => applyWhitespaceProcessing(p.trim(), whitespaceProcessingMethod))
 	paragraphs = paragraphs.filter(p => p.length > 0)
 
 	return paragraphs
 }
 
+export function applyWhitespaceProcessing(text: string, whitespaceProcessingMethod: WhitespaceProcessing) {
+	if (whitespaceProcessingMethod === 'removeLineBreaks') {
+		return text.replaceAll(/(\r?\n)+/g, ' ')
+	} else if (whitespaceProcessingMethod === 'collapse') {
+		return text.replaceAll(/\s+/g, ' ')
+	} else if (whitespaceProcessingMethod === 'preserve') {
+		return text
+	} else {
+		throw new Error(`Invalid whitespace processing method: '${whitespaceProcessingMethod}'`)
+	}
+}
+
 export function splitToLines(text: string) {
 	return text.split(/\r?\n/g)
 }