echogarden 2.7.0 → 2.8.0

package/src/subtitles/Subtitles.ts

@@ -6,7 +6,8 @@ import { charactersToWriteAhead } from '../audio/AudioPlayer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { readFileAsUtf8 } from '../utilities/FileSystem.js'
 import { deepClone } from '../utilities/ObjectUtilities.js'
-import { formatHMS, formatMS, startsWithAnyOf } from '../utilities/StringUtilities.js'
+import { formatHMS, formatMS } from '../utilities/StringUtilities.js'
+import { anyOf, buildRegExp, capture, digit, inputStart, matches, notUnicodeProperty, oneOrMore, zeroOrMore } from 'regexp-composer'
 
 export async function subtitlesFileToText(filename: string) {
 	return subtitlesToText(await readFileAsUtf8(filename))
@@ -257,14 +258,12 @@ function getCuesFromTimeline_IsolateSegmentSentence(timeline: Timeline, config:
 			const lineLengthWithNextWord = nextWordExtendedEndOffset - lineStartOffset
 			const wordsRemaining = wordTimeline.length - wordIndex - 1
 
-			const phraseSeparators = [',', '，', '、', ';', ':', '),', '",', '”,', '.', '".', '”.', '."', '.”', '。']
-
 			const lineLengthWithNextWordExceedsMaxLineWidth = lineLengthWithNextWord >= maxLineWidth
 			const lineLengthExceedsHalfMaxLineWidth = lineLength >= maxLineWidth / 2
 
 			const wordsRemainingAreEqualOrLessToMinimumWordsInLine = wordsRemaining <= config.minWordsInLine!
 			const remainingTextExceedsMaxLineWidth = entryText.length - lineStartOffset > maxLineWidth
-			const followingSubstringIsPhraseSeparator = startsWithAnyOf(entryText.substring(wordEndOffset), phraseSeparators)
+			const followingSubstringIsPhraseSeparator = phraseSeparatorRegExp.test(entryText.substring(wordEndOffset))
 
 			const shouldAddNewLine =
 				isLastWord ||
@@ -434,8 +433,7 @@ function getCuesFromTimeline_IsolateLines(timeline: Timeline, config: SubtitlesC
 }
 
 export function tryParseTimeRangePatternWithHours(line: string) {
-	const timeRangePatternWithHours = /^(\d+)\:(\d+)\:(\d+)[\.,](\d+)[ ]*-->[ ]*(\d+)\:(\d+)\:(\d+)[\.,](\d+)/
-	const match = timeRangePatternWithHours.exec(line)
+	const match = timeRangeWithHoursRegExp.exec(line)
 
 	if (!match) {
 		return { startTime: -1, endTime: -1, succeeded: false }
@@ -458,8 +456,7 @@ export function tryParseTimeRangePatternWithHours(line: string) {
 }
 
 export function tryParseTimeRangePatternWithoutHours(line: string) {
-	const timeRangePatternWithHours = /^(\d+)\:(\d+)[\.,](\d+)[ ]*-->[ ]*(\d+)\:(\d+)[\.,](\d+)/
-	const match = timeRangePatternWithHours.exec(line)
+	const match = timeRangeWithoutHoursRegExp.exec(line)
 
 	if (!match) {
 		return { startTime: -1, endTime: -1, succeeded: false }
@@ -514,6 +511,70 @@ function cueObjectToText(cue: Cue, cueIndex: number, config: SubtitlesConfig) {
 	return outText
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Patterns
+////////////////////////////////////////////////////////////////////////////////////////////////////////
+const timeRangeWithHoursRegExp = buildRegExp([
+	inputStart,
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+
+	anyOf('.', ','),
+	capture(oneOrMore(digit)),
+
+	zeroOrMore(' '),
+	'-->',
+	zeroOrMore(' '),
+
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+
+	anyOf('.', ','),
+	capture(oneOrMore(digit)),
+])
+
+const timeRangeWithoutHoursRegExp = buildRegExp([
+	inputStart,
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+
+	anyOf('.', ','),
+	capture(oneOrMore(digit)),
+
+	zeroOrMore(' '),
+	'-->',
+	zeroOrMore(' '),
+
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+
+	anyOf('.', ','),
+	capture(oneOrMore(digit)),
+])
+
+const phraseSeparatorCharacters = [',', '，', '、', ';', ':', '),', '",', '”,']
+
+const phraseSeparatorRegExp = buildRegExp(
+	matches([
+		inputStart,
+		anyOf(...phraseSeparatorCharacters)
+	], {
+		ifPrecededBy: notUnicodeProperty('Decimal_Number'),
+		ifFollowedBy: notUnicodeProperty('Decimal_Number'),
+	})
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Types
+////////////////////////////////////////////////////////////////////////////////////////////////////////
 export type Cue = {
 	lines: string[]
 	startTime: number

package/src/synthesis/ElevenLabsTTS.ts

@@ -5,7 +5,7 @@ import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { decodeBase64 } from '../encodings/Base64.js'
-import { isWordOrSymbolWord, splitToWords } from '../nlp/Segmentation.js'
+import { splitToWords } from '../nlp/Segmentation.js'
 import { Timeline } from '../utilities/Timeline.js'
 
 const log = logToStderr
@@ -79,24 +79,25 @@ export async function synthesize(text: string, voiceId: string, language: string
 		logger.start('Create timeline from returned character timings')
 
 		const referenceText = characters.join('')
-		const words = (await splitToWords(referenceText, language)).filter(w => isWordOrSymbolWord(w))
+		const wordSequence = await splitToWords(referenceText, language)
 
 		timeline = []
 
-		let offset = 0
+		for (const wordEntry of wordSequence.entries) {
+			const wordText = wordEntry.text
 
-		for (const word of words) {
-			const wordStartIndex = referenceText.indexOf(word, offset)
-			const wordEndIndex = wordStartIndex + word.length
+			const wordStartOffset = wordEntry.startOffset
+			const wordEndOffset = wordEntry.endOffset
+
+			const startTime = characterStartTimes[wordStartOffset]
+			const endTime = characterEndTimes[wordEndOffset] ?? characterEndTimes[wordEndOffset - 1]
 
 			timeline.push({
 				type: 'word',
-				text: word,
-				startTime: characterStartTimes[wordStartIndex],
-				endTime: characterEndTimes[wordEndIndex] ?? characterEndTimes[wordEndIndex - 1]
+				text: wordText,
+				startTime,
+				endTime,
 			})
-
-			offset = wordEndIndex
 		}
 	}
 

package/src/synthesis/EspeakTTS.ts

@@ -1,17 +1,18 @@
 import { concatFloat32Arrays, logToStderr, formatObjectToString } from '../utilities/Utilities.js'
 import { int16PcmToFloat32 } from '../audio/AudioBufferConversion.js'
 import { Logger } from '../utilities/Logger.js'
-import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js'
 import { RawAudio, getEmptyRawAudio } from '../audio/AudioUtilities.js'
 import { getNormalizedFragmentsForSpeech, simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
 import { ipaPhoneToKirshenbaum } from '../nlp/PhoneConversion.js'
-import { splitToWords, wordCharacterPattern } from '../nlp/Segmentation.js'
+import { splitToWords, wordCharacterRegExp } from '../nlp/Segmentation.js'
 import { Lexicon, tryGetFirstLexiconSubstitution } from '../nlp/Lexicon.js'
 import { phonemizeSentence } from '../nlp/EspeakPhonemizer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { escapeHtml } from '../encodings/HtmlEscape.js'
 
+import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
+
 const log = logToStderr
 
 let espeakInstance: any
@@ -37,7 +38,7 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 	fragments = []
 	preprocessedFragments = []
 
-	let words = await splitToWords(text, language)
+	let words = (await splitToWords(text, language)).wordArray
 
 	// Merge repeating non-words to a single word to work around eSpeak bug
 	{
@@ -51,7 +52,7 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 				i > 0 &&
 				currentWord === previousWord &&
 				!['[', ']'].includes(currentWord) && // Work around eSpeak-NG marker bug with repeating squared brackets
-				!wordCharacterPattern.test(currentWord)) {
+				!wordCharacterRegExp.test(currentWord)) {
 
 				wordsWithMerges[wordsWithMerges.length - 1] += currentWord
 			} else {
@@ -515,12 +516,12 @@ export async function textToPhonemes(text: string, voice: string, useIPA = true)
 	const { instance, module } = await getEspeakInstance()
 	const textPtr = instance.convert_to_phonemes(text, useIPA)
 
-	const wasmMemory = new WasmMemoryManager(module)
+	const wasmHeap = wrapEmscriptenModuleHeap(module)
 
-	const resultRef = wasmMemory.wrapNullTerminatedUtf8String(textPtr.ptr)
-	const result = resultRef.getValue()
+	const resultRef = wasmHeap.wrapNullTerminatedUtf8String(textPtr.ptr)
+	const result = resultRef.value
 
-	wasmMemory.freeAll()
+	resultRef.free()
 
 	return result
 }

package/src/synthesis/KokoroTTS.ts

@@ -108,9 +108,6 @@ export class KokoroTTS {
 		const {
 			referenceSynthesizedAudio,
 			referenceTimeline,
-			fragments,
-			phonemizedFragmentsSubstitutions,
-			phonemizedSentence
 		} = await Espeak.preprocessAndSynthesize(sentenceText, voiceLanguage, espeakOptions, lexicons)
 
 		logger.end()
@@ -132,13 +129,28 @@ export class KokoroTTS {
 
 		const allTokenIds: number[] = []
 
-		for (let phraseIndex = 0; phraseIndex < phonemizedSentence.length; phraseIndex++) {
-			const phrase = phonemizedSentence[phraseIndex]
+		for (let clauseIndex = 0; clauseIndex < referenceTimeline.length; clauseIndex++) {
+			const clauseEntry = referenceTimeline[clauseIndex]
 
-			for (let wordIndex = 0; wordIndex < phrase.length; wordIndex++) {
-				const word = phrase[wordIndex]
+			if (!clauseEntry.timeline) {
+				continue
+			}
+
+			const wordTimelineForClause = clauseEntry.timeline
+
+			for (let wordIndex = 0; wordIndex < wordTimelineForClause.length; wordIndex++) {
+				const wordEntry = wordTimelineForClause[wordIndex]
+				const wordText = wordEntry.text
+
+				if (!wordEntry.timeline) {
+					continue
+				}
+
+				const tokenTimelineForWord = wordEntry.timeline
+				const phonemeTimelineForWord = tokenTimelineForWord.flatMap(tokenEntry => tokenEntry.timeline ?? [])
 
-				for (const phoneme of word) {
+				for (const phonemeEntry of phonemeTimelineForWord) {
+					const phoneme = phonemeEntry.text
 					let processedPhoneme = phoneme
 
 					if (voicePrimaryLanguageShort === 'en') {
@@ -179,7 +191,7 @@ export class KokoroTTS {
 
 						if (false) {
 							// Workaround a word having only 'I' not being pronounced at some cases
-							if (processedPhoneme === 'I' && word.length === 1) {
+							if (processedPhoneme === 'I' && wordText.length === 1) {
 								processedPhoneme = 'aɪ'
 								//processedPhoneme = 'ˌI'
 							}
@@ -196,10 +208,10 @@ export class KokoroTTS {
 					}
 				}
 
-				if (wordIndex < phrase.length - 1) {
+				if (wordIndex < wordTimelineForClause.length - 1) {
 					allTokenIds.push(wordBreakTokenId)
 				} else {
-					if (phraseIndex < phonemizedSentence.length - 1) {
+					if (clauseIndex < referenceTimeline.length - 1) {
 						allTokenIds.push(phraseBreakTokenId)
 						allTokenIds.push(wordBreakTokenId)
 					}
@@ -209,7 +221,7 @@ export class KokoroTTS {
 
 		allTokenIds.push(sentenceEndTokenId)
 
-		const maxPartLength = 509
+		const maxPartLength = 500
 
 		const parts: number[][] = []
 

package/src/synthesis/SvoxPicoTTS.ts

@@ -2,11 +2,12 @@ import { SynthesisVoice } from '../api/API.js'
 import { decodeToChannels, SampleFormat } from '../audio/AudioBufferConversion.js'
 import { bandwidthToQFactor } from '../dsp/BiquadFilter.js'
 import { Logger } from '../utilities/Logger.js'
-import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
 import { readFileAsBinary } from '../utilities/FileSystem.js'
 import { concatUint8Arrays } from '../utilities/Utilities.js'
 
+import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
+
 let svoxPicoInstance: any
 
 export async function synthesize(text: string, textAnalysisFilePath: string, signalGenerationFilePath: string, postprocessOutput = true) {
@@ -17,7 +18,7 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 
 	logger.start('Initialize pico engine')
 
-	const wasmMemory = new WasmMemoryManager(m)
+	const wasmHeap = wrapEmscriptenModuleHeap(m)
 
 	const pico_initialize = m._pico_initialize
 	const picoext_setTraceLevel = m._picoext_setTraceLevel
@@ -37,11 +38,11 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 	const pico_unloadResource = m._pico_unloadResource
 
 	const picoMemSize = 2500000
-	const picoMemAreaRef = wasmMemory.allocUint8Array(picoMemSize)
+	const picoMemAreaRef = wasmHeap.allocUint8Array(picoMemSize)
 
-	const systemPtrRef = wasmMemory.allocPointer()
+	const systemPtrRef = wasmHeap.allocPointer32()
 
-	let resultCode = pico_initialize(picoMemAreaRef.address, picoMemAreaRef.length, systemPtrRef.address)
+	let resultCode = pico_initialize(picoMemAreaRef.address, picoMemAreaRef.allocatedByteCount, systemPtrRef.address)
 	const systemPtr = systemPtrRef.value
 
 	throwErrorIfFailed(resultCode, 'Failed Pico initialization.')
@@ -54,8 +55,8 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 		const fileData = await readFileAsBinary(localFilePath)
 		m.FS.writeFile(virtualFilePath, fileData)
 
-		const virtualFilePathRef = wasmMemory.allocNullTerminatedUtf8String(virtualFilePath)
-		const resourcePtrRef = wasmMemory.allocPointer()
+		const virtualFilePathRef = wasmHeap.allocNullTerminatedUtf8String(virtualFilePath)
+		const resourcePtrRef = wasmHeap.allocPointer32()
 
 		resultCode = pico_loadResource(systemPtr, virtualFilePathRef.address, resourcePtrRef.address)
 		const resourcePtr = resourcePtrRef.value
@@ -69,12 +70,12 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 	const { resourcePtr: signalGenerationResourcePtr, resourcePtrRef: signalGenerationResourcePtrRef } = await loadResource(signalGenerationFilePath)
 
 	function getResourceName(resourcePtr: number) {
-		const resourceNameRef = wasmMemory.allocUint8Array(32)
+		const resourceNameRef = wasmHeap.allocNullTerminatedUtf8String(32)
 		resultCode = pico_getResourceName(systemPtr, resourcePtr, resourceNameRef.address)
 
 		throwErrorIfFailed(resultCode, `Failed getting Pico resource name.`)
 
-		const resourceName = resourceNameRef.readAsNullTerminatedUtf8String()
+		const resourceName = resourceNameRef.value
 
 		return { resourceName, resourceNameRef }
 	}
@@ -82,7 +83,7 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 	const { resourceName: textAnalysisResourceName, resourceNameRef: textAnalysisResourceNameRef } = getResourceName(textAnalysisResourcePtr)
 	const { resourceName: signalGenerationResourceName, resourceNameRef: signalGenerationResourceNameRef } = getResourceName(signalGenerationResourcePtr)
 
-	const voiceNameRef = wasmMemory.allocNullTerminatedUtf8String('PicoVoice')
+	const voiceNameRef = wasmHeap.allocNullTerminatedUtf8String('PicoVoice')
 
 	resultCode = pico_createVoiceDefinition(systemPtr, voiceNameRef.address)
 
@@ -95,7 +96,7 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 	addResourceToVoiceDefinition(textAnalysisResourceNameRef.address)
 	addResourceToVoiceDefinition(signalGenerationResourceNameRef.address)
 
-	const enginePtrRef = wasmMemory.allocPointer()
+	const enginePtrRef = wasmHeap.allocPointer32()
 	resultCode = pico_newEngine(systemPtr, voiceNameRef.address, enginePtrRef.address)
 
 	throwErrorIfFailed(resultCode, `Failed creating new engine.`)
@@ -104,15 +105,17 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 
 	logger.start('Synthesize with pico')
 
-	const textRef = wasmMemory.allocNullTerminatedUtf8String(text)
+	const textRef = wasmHeap.allocNullTerminatedUtf8String(text)
 
-	const bytesWrittenRef = wasmMemory.allocInt32()
+	const bytesWrittenRef = wasmHeap.allocInt32()
 
 	const audioParts: Uint8Array[] = []
 
-	for (let textByteOffset = 0; textByteOffset < textRef.length;) {
+	const textRefEncodedByteCountIncludingTerminator = textRef.encodedByteCount + 1
+
+	for (let textByteOffset = 0; textByteOffset < textRefEncodedByteCountIncludingTerminator;) {
 		bytesWrittenRef.value = 0
-		resultCode = pico_putTextUtf8(enginePtr, textRef.address + textByteOffset, textRef.length - textByteOffset, bytesWrittenRef.address)
+		resultCode = pico_putTextUtf8(enginePtr, textRef.address + textByteOffset, textRefEncodedByteCountIncludingTerminator - textByteOffset, bytesWrittenRef.address)
 		const bytesWritten = bytesWrittenRef.value
 
 		throwErrorIfFailed(resultCode, `Failed writing text to engine.`)
@@ -129,13 +132,13 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 		const outBuffers: Uint8Array[] = []
 
 		const outBufferLength = 16384
-		const outBufferRef = wasmMemory.allocUint8Array(outBufferLength)
+		const outBufferRef = wasmHeap.allocUint8Array(outBufferLength)
 
-		const outByteCountRef = wasmMemory.allocInt16()
-		const outDataTypeRef = wasmMemory.allocInt16()
+		const outByteCountRef = wasmHeap.allocInt16()
+		const outDataTypeRef = wasmHeap.allocInt16()
 
 		while (true) {
-			resultCode = pico_getData(enginePtr, outBufferRef.address, outBufferRef.length, outByteCountRef.address, outDataTypeRef.address)
+			resultCode = pico_getData(enginePtr, outBufferRef.address, outBufferRef.allocatedByteCount, outByteCountRef.address, outDataTypeRef.address)
 
 			throwErrorIfFailed(resultCode, `Failed getting audio data from engine.`, [200, 201])
 
@@ -147,7 +150,7 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 			}
 
 			if (outByteCount > 0) {
-				outBuffers.push(outBufferRef.slice(0, outByteCount) as Uint8Array)
+				outBuffers.push(outBufferRef.view.slice(0, outByteCount))
 			}
 		}
 
@@ -179,7 +182,7 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 
 		pico_terminate(systemPtrRef.address)
 
-		wasmMemory.freeAll()
+		wasmHeap.freeAll()
 	}
 
 	function throwErrorIfFailed(resultCode: number, title: string, successCodes = [0]) {
@@ -187,9 +190,9 @@ export async function synthesize(text: string, textAnalysisFilePath: string, sig
 			return
 		}
 
-		const picoErrorMessageRef = wasmMemory.allocUint8Array(200)
+		const picoErrorMessageRef = wasmHeap.allocNullTerminatedUtf8String(200)
 		pico_getSystemStatusMessage(systemPtr, resultCode, picoErrorMessageRef)
-		const picoErrorMessage = picoErrorMessageRef.readAsNullTerminatedUtf8String()
+		const picoErrorMessage = picoErrorMessageRef.value
 
 		dispose()
 		throw new Error(`${title} ${picoErrorMessage}`)

package/src/text-translation/DeepLTextTranslation.ts

@@ -1,6 +1,6 @@
 import { request } from "gaxios"
 import { Logger } from "../utilities/Logger.js"
-import { splitToSentences } from "../nlp/Segmentation.js"
+import { parseText } from "../nlp/Segmentation.js"
 import { TranslationPair } from "../api/TextTranslation.js"
 import { getChromeOnWindowsHeaders } from "../utilities/BrowserRequestHeaders.js"
 
@@ -9,7 +9,8 @@ export async function translateText(text: string, sourceLanguage: string, target
 
 	logger.start(`Prepare request`)
 
-	const textSentences = splitToSentences(text, sourceLanguage)
+	const segmentedText = await parseText(text, sourceLanguage)
+	const sentences = segmentedText.sentences
 
 	const requestBodyObject = {
 		jsonrpc: '2.0',
@@ -20,7 +21,7 @@ export async function translateText(text: string, sourceLanguage: string, target
 					kind: 'default',
 					sentences: [
 						{
-							text: textSentences[0],
+							text: sentences[0].text,
 							id: 1,
 							prefix: '',
 						}

package/src/text-translation/NLLBTextTranslation.ts

@@ -1,6 +1,6 @@
 import chalk from 'chalk'
 import { TranslationPair } from '../api/TextTranslation.js'
-import { splitToSentences } from '../nlp/Segmentation.js'
+import { parseText } from '../nlp/Segmentation.js'
 import { Logger } from '../utilities/Logger.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 
@@ -30,19 +30,21 @@ export async function translateText(sourceText: string, sourceLanguage: string,
 		tgt_lang: 'eng_Latn'
 	}
 
-	const sentences = splitToSentences(sourceText, sourceLanguage)
+	const segmentedText = await parseText(sourceText, sourceLanguage)
+
+	const sentences = segmentedText.sentences
 
 	logger.end()
 
 	const translationPairs: TranslationPair[] = []
 
 	for (let i = 0; i < sentences.length; i++) {
-		const sentence = sentences[i]
+		const sentenceText = sentences[i].text
 
-		logger.logTitledMessage(`Translate sentence ${i + 1}/${sentences.length}`, `"${sentence.trim()}"`, chalk.magentaBright)
+		logger.logTitledMessage(`Translate sentence ${i + 1}/${sentences.length}`, `"${sentenceText.trim()}"`, chalk.magentaBright)
 
 		logger.start(`Tokenize sentence`)
-		const inputs = (tokenizer as any)._build_translation_inputs(sentence, {
+		const inputs = (tokenizer as any)._build_translation_inputs(sentenceText, {
 			padding: true,
 			truncation: true,
 		}, config)
@@ -67,7 +69,7 @@ export async function translateText(sourceText: string, sourceLanguage: string,
 			.trim()
 
 		translationPairs.push({
-			sourceText: sentence,
+			sourceText: sentenceText,
 			translatedText
 		})
 

package/src/typings/Fillers.d.ts

@@ -1,5 +1,3 @@
-declare module 'moving-median'
-
 declare module 'html-to-text' {
 	export function htmlToText(html: string, options: {
 		wordwrap: boolean | number
@@ -7,20 +5,6 @@ declare module 'html-to-text' {
 	})
 }
 
-declare module 'cldr-segmentation' {
-	export function sentenceSplit(text: string, suppressions: any)
-	export function wordSplit(text: string, suppressions: any)
-	export const suppressions: Record<string, Suppressions>
-
-	export class Suppressions {
-		constructor(forwardTrie: any, backwardTrie: any, list: string)
-
-		merge(other: Suppressions)
-
-		static create(list: string[]): Suppressions
-	}
-}
-
 declare module 'winax'
 
 declare module 'tinyld' {

package/src/utilities/StringUtilities.ts

@@ -52,7 +52,7 @@ export function formatListWithQuotedElements(strings: string[], quoteSymbol = `'
 
 export function getUTF32Chars(str: string) {
 	const utf32chars: string[] = []
-	const mapping: number[] = []
+	const utf16To32Mapping: number[] = []
 
 	let utf32Index = 0
 
@@ -60,15 +60,15 @@ export function getUTF32Chars(str: string) {
 		utf32chars.push(utf32char)
 
 		for (let i = 0; i < utf32char.length; i++) {
-			mapping.push(utf32Index)
+			utf16To32Mapping.push(utf32Index)
 		}
 
 		utf32Index += 1
 	}
 
-	mapping.push(utf32Index)
+	utf16To32Mapping.push(utf32Index)
 
-	return { utf32chars, mapping }
+	return { utf32chars, utf16To32Mapping }
 }
 
 export function containsInvalidCodepoint(str: string) {
@@ -81,10 +81,9 @@ export function containsInvalidCodepoint(str: string) {
 	return false
 }
 
-
 export function splitAndPreserveSeparators(text: string, separatorRegex: RegExp): string[] {
 	if (!separatorRegex.flags.includes('g')) {
-		throw new Error('RegExp must be global')
+		throw new Error('Separator regular expression must have a global flag')
 	}
 
 	// Use the match method to find all matches for the separators