echogarden 3.0.5 → 3.2.0

package/src/encodings/Utf8.ts

@@ -1,5 +1,8 @@
-import { EncodeIntoResult } from "./TextEncodingsCommon.js"
+import { EncodeIntoResult } from './TextEncodingsCommon.js'
 
+//////////////////////////////////////////////////////////////////////////////
+// UTF-8 Encoding
+//////////////////////////////////////////////////////////////////////////////
 export function encodeUtf8(text: string) {
 	const textEncoder = new TextEncoder()
 
@@ -14,104 +17,64 @@ export function encodeUtf8Into(text: string, outputArray: Uint8Array): EncodeInt
 	return result
 }
 
-export function decodeUtf8(encodedString: Uint8Array) {
-	const maxChunkLength = 2 ** 24
+export class ChunkedUtf8Encoder {
+	private readonly textEncoder = new TextEncoder()
 
-	const decoder = new ChunkedUtf8Decoder()
+	private pendingHighSurrogate = ''
 
-	for (let offset = 0; offset < encodedString.length; offset += maxChunkLength) {
-		const chunk = encodedString.subarray(offset, offset + maxChunkLength)
+	writeChunk(stringChunk: string): Uint8Array {
+		if (this.pendingHighSurrogate !== '') {
+			stringChunk = this.pendingHighSurrogate + stringChunk
 
-		decoder.writeChunk(chunk)
-	}
+			this.pendingHighSurrogate = ''
+		}
 
-	return decoder.toString()
-}
+		const lastCodeUnit = stringChunk.charCodeAt(stringChunk.length - 1)
 
-export class ChunkedUtf8Decoder {
-	private str = ''
-	private readonly textDecoder = new TextDecoder('utf-8')
+		if (lastCodeUnit >= 0xD800 && lastCodeUnit <= 0xDBFF) {
+			this.pendingHighSurrogate = stringChunk[stringChunk.length - 1]
 
-	writeChunk(chunk: Uint8Array) {
-		const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
+			stringChunk = stringChunk.substring(0, stringChunk.length - 1)
+		}
 
-		this.str += decodedChunk
+		return this.textEncoder.encode(stringChunk)
 	}
 
-	toString() {
-		return this.str
+	finalize(): Uint8Array {
+		const result = this.textEncoder.encode(this.pendingHighSurrogate)
+
+		this.pendingHighSurrogate = ''
+
+		return result
 	}
 }
 
-//////////////////////////////////////////////////////////////////////////////////////////////
-// Pure JavaScript implementations
-//////////////////////////////////////////////////////////////////////////////////////////////
-function encodeUtf8Into_JS(str: string, outputArray: Uint8Array): EncodeIntoResult {
-	let readOffset = 0
-	let writeOffset = 0
-
-	while (readOffset < str.length) {
-		const charCode = str.codePointAt(readOffset++)!
-
-		if (charCode <= 0x7f) {
-			outputArray[writeOffset++] = charCode
-		} else if (charCode <= 0x7ff) {
-			outputArray[writeOffset++] = 0xc0 | (charCode >>> 6)
-			outputArray[writeOffset++] = 0x80 | (charCode & 63)
-		} else if (charCode <= 0xffff) {
-			outputArray[writeOffset++] = 0xe0 | (charCode >>> 12)
-			outputArray[writeOffset++] = 0x80 | ((charCode >>> 6) & 63)
-			outputArray[writeOffset++] = 0x80 | (charCode & 63)
-		} else if (charCode <= 0x10ffff) {
-			outputArray[writeOffset++] = 0xf0 | (charCode >>> 18)
-			outputArray[writeOffset++] = 0x80 | ((charCode >>> 12) & 63)
-			outputArray[writeOffset++] = 0x80 | ((charCode >>> 6) & 63)
-			outputArray[writeOffset++] = 0x80 | (charCode & 63)
-
-			readOffset++
-		}
+//////////////////////////////////////////////////////////////////////////////
+// UTF-8 Decoding
+//////////////////////////////////////////////////////////////////////////////
+export function decodeUtf8(utf8Bytes: Uint8Array) {
+	const maxChunkLength = 2 ** 24
+
+	const chunkedUtf8Decoder = new ChunkedUtf8Decoder()
+
+	let resultString = ''
+
+	for (let offset = 0; offset < utf8Bytes.length; offset += maxChunkLength) {
+		const utf8Chunk = utf8Bytes.subarray(offset, offset + maxChunkLength)
+		const stringChunk = chunkedUtf8Decoder.writeChunk(utf8Chunk)
+
+		resultString += stringChunk
 	}
 
-	return { read: str.length, written: writeOffset }
+	return resultString
 }
 
-function decodeUtf8_JS(utf8Bytes: Uint8Array): string {
-	let decodedString = ''
-
-	let readOffset = 0
-
-	while (readOffset < utf8Bytes.length) {
-		const leadByte = utf8Bytes[readOffset++]
-
-		let outputCodePoint: number
-
-		if (leadByte >>> 7 === 0) {
-			outputCodePoint = leadByte
-		} else if (leadByte >>> 5 === 6) {
-			outputCodePoint =
-				(leadByte & 31) << 6 |
-				(utf8Bytes[readOffset++] & 63)
-		} else if (leadByte >>> 4 === 14) {
-			outputCodePoint =
-				(leadByte & 15) << 12 |
-				(utf8Bytes[readOffset++] & 63) << 6 |
-				(utf8Bytes[readOffset++] & 63)
-		} else if (leadByte >>> 3 === 30) {
-			outputCodePoint =
-				(leadByte & 7) << 18 |
-				(utf8Bytes[readOffset++] & 63) << 12 |
-				(utf8Bytes[readOffset++] & 63) << 6 |
-				(utf8Bytes[readOffset++] & 63)
-		} else {
-			throw new Error(`Invalid UTF-8 stream: An invalid lead byte value encountered at position ${readOffset}`)
-		}
+export class ChunkedUtf8Decoder {
+	private readonly textDecoder = new TextDecoder('utf-8')
 
-		decodedString += String.fromCodePoint(outputCodePoint)
-	}
+	writeChunk(chunk: Uint8Array) {
+		const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
 
-	if (readOffset > utf8Bytes.length) {
-		throw new Error(`UTF-8 decoding failed. Byte sequence is truncated.`)
+		return decodedChunk
 	}
-
-	return decodedString
 }

package/src/nlp/Segmentation.ts

@@ -1,7 +1,8 @@
 import { sumArray, logToStderr } from '../utilities/Utilities.js'
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
-import { includesAnyOf, splitAndPreserveSeparators } from '../utilities/StringUtilities.js'
+import { splitAndPreserveSeparators } from '../utilities/StringUtilities.js'
+import { anyOf, buildRegExp, unicodeProperty, oneOrMore, possibly, codepoint, inputStart, inputEnd, whitespace } from 'regexp-composer'
 
 import * as TextSegmentation from '@echogarden/text-segmentation'
 import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js'
@@ -9,50 +10,81 @@ import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js'
 
 const log = logToStderr
 
-export const wordCharacterRegExp = /[\p{Letter}\p{Number}]/u
+const includesWordCharacterPattern = anyOf(unicodeProperty('Letter'), unicodeProperty('Number'))
+const includesWordCharacterRegExp = buildRegExp(includesWordCharacterPattern)
 
 // See: https://mathiasbynens.be/notes/es-unicode-property-escapes
-export const emojiSequenceRegExp = /\p{Emoji_Modifier_Base}\p{Emoji_Modifier}?|\p{Emoji_Presentation}|\p{Emoji}\uFE0F/u
+//export const emojiSequenceRegExp = /\p{Emoji_Modifier_Base}\p{Emoji_Modifier}?|\p{Emoji_Presentation}|\p{Emoji}\uFE0F/u
+const includesEmojiSequencePattern = anyOf(
+	[unicodeProperty('Emoji_Modifier_Base'), possibly(unicodeProperty('Emoji_Modifier'))],
+	unicodeProperty('Emoji_Presentation'),
+	[unicodeProperty('Emoji'), codepoint('FE0F')]
+)
+const includesEmojiSequenceRegExp = buildRegExp(includesEmojiSequencePattern)
 
-export const includesPunctuationRegExp = /[\p{Punctuation}]/u
-export const isAllPunctuationRegExp = /^[\p{Punctuation}]+$/u
+const symbolWordsList = [
+	'$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷',
+	'/', '\\', '^', '*', '×', '=', '≈', '¼', '½', '¾', '→', '≤', '≥', '∞'
+]
 
-export const phraseSeparators = [',', '、', '，', '،', ';', '；', ':', '：', '—']
-export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '\\', '^', '*', '×', '=', '≈', '¼', '½', '¾', '→', '≤', '≥']
+const includesSymbolWordPattern = anyOf(...symbolWordsList)
+const includesSymbolWordRegExp = buildRegExp(includesSymbolWordPattern)
+
+const isAllSymbolWordsPattern = [inputStart, oneOrMore(includesSymbolWordPattern), inputEnd]
+const isAllSymbolWordsRegExp = buildRegExp(isAllSymbolWordsPattern)
+
+const includesWordCharacterOrEmojiPattern = anyOf(includesWordCharacterPattern, includesEmojiSequencePattern)
+const includesWordCharacterOrEmojiRegExp = buildRegExp(includesWordCharacterOrEmojiPattern)
+
+const includesWordCharacterOrEmojiOrIsAllSymbolWordPattern = anyOf(includesWordCharacterPattern, includesEmojiSequencePattern, isAllSymbolWordsPattern)
+const includesWordCharacterOrEmojiOrIsAllSymbolWordRegExp = buildRegExp(includesWordCharacterOrEmojiOrIsAllSymbolWordPattern)
+
+const includesPunctuationPattern = unicodeProperty('Punctuation')
+const includesPunctuationRegExp = buildRegExp(includesPunctuationPattern)
+
+const isAllPunctuationPattern = [inputStart, oneOrMore(includesPunctuationPattern), inputEnd]
+const isAllPunctuationRegExp = buildRegExp(isAllPunctuationPattern)
+
+const phraseSeparatorsList = [',', '、', '，', '،', ';', '；', ':', '：', '—']
+const includesPhraseSeparatorsPattern = anyOf(...phraseSeparatorsList)
+const includesPhraseSeparatorsRegExp = buildRegExp(includesPhraseSeparatorsPattern)
+
+const isAllWhitespacePattern = [inputStart, oneOrMore(whitespace), inputEnd]
+const isAllWhitespaceRegExp = buildRegExp(isAllWhitespacePattern)
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
 // Predicates
 ///////////////////////////////////////////////////////////////////////////////////////////////
 export function isWordOrEmojiOrSymbolWord(str: string) {
-	return isWordOrEmoji(str) || symbolWords.includes(str)
+	return includesWordCharacterOrEmojiOrIsAllSymbolWordRegExp.test(str.trim())
 }
 
 export function isWordOrEmoji(str: string) {
-	return isWord(str) || includesEmoji(str)
+	return includesWordCharacterOrEmojiRegExp.test(str.trim())
 }
 
 export function isSymbolWord(str: string) {
-	return symbolWords.includes(str?.trim())
+	return isAllSymbolWordsRegExp.test(str.trim())
 }
 
-export function isWord(str: string) {
-	return wordCharacterRegExp.test(str?.trim())
+export function includesWordCharacter(str: string) {
+	return includesWordCharacterRegExp.test(str.trim())
 }
 
 export function includesPunctuation(str: string) {
-	return includesPunctuationRegExp.test(str?.trim())
+	return includesPunctuationRegExp.test(str.trim())
 }
 
 export function isAllPunctuation(str: string) {
-	return isAllPunctuationRegExp.test(str?.trim())
+	return isAllPunctuationRegExp.test(str.trim())
 }
 
 export function includesEmoji(str: string) {
-	return emojiSequenceRegExp.test(str?.trim())
+	return includesEmojiSequenceRegExp.test(str.trim())
 }
 
 export function isAllWhitespace(str: string) {
-	return str && /^\s+$/.test(str)
+	return isAllWhitespaceRegExp.test(str)
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
@@ -280,11 +312,13 @@ export class Word {
 		this.isSentenceFinalizer = isSentenceFinalizer
 	}
 
-	get containsOnlyPunctuation() { return !wordCharacterRegExp.test(this.text) && !this.isSymbolWord }
+	get containsOnlyPunctuation() { return !isWordOrEmojiOrSymbolWord(this.text) }
 
-	get isSymbolWord() { return symbolWords.includes(this.text) }
+	get isSymbolWord() { return isSymbolWord(this.text) }
 
-	get isPhraseSeperator() { return this.containsOnlyPunctuation && includesAnyOf(this.text, phraseSeparators) }
+	get isPhraseSeperator() {
+		return this.containsOnlyPunctuation && includesPhraseSeparatorsRegExp.test(this.text)
+	}
 
 	get length() { return this.text.length }
 }

package/src/nlp/TextNormalizer.ts

@@ -1,6 +1,6 @@
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { substituteCharactersUsingLookup } from '../utilities/StringUtilities.js'
-import { anyOf, buildRegExp, charRange, inputEnd, inputStart, repeated, zeroOrMore } from 'regexp-composer'
+import { anyOf, buildRegExp, charRange, inputEnd, inputStart, oneOrMore, repeated, unicodeProperty, zeroOrMore } from 'regexp-composer'
 
 export function getNormalizedFragmentsForSpeech(
 	words: string[],
@@ -25,14 +25,24 @@ export function getNormalizedFragmentsForSpeech(
 		const nextNonWhitespaceWord = nextNonWhitespaceWords[0]
 
 		const originalWordIndex = nonWhitespaceWordOriginalIndex[wordIndex]
+
 		const isFollowedByWhitespace = words[originalWordIndex + 1]?.trim().length === 0
+		const isSpecialCharacterBeforeYear = ['(', ',', '©'].includes(words[originalWordIndex])
 
-		if (wordsPrecedingAYear.includes(lowerCaseWord) &&
-			isFollowedByWhitespace &&
-			fourDigitYearPatternRegExp.test(nextNonWhitespaceWord)) {
+		//const isWordPrecedingAYear = wordsPrecedingAYear.includes(lowerCaseWord)
+		const isWordPrecedingAYear =
+			isAllLettersOrApostropheRegExp.test(lowerCaseWord) || isSpecialCharacterBeforeYear
+
+		const followedByFourDigitYearPattern = fourDigitYearRegExp.test(nextNonWhitespaceWord)
 
- 			// Normalize a four digit year pattern, e.g. 'in 1995'.
+		const isWordPrecedingADecade = wordsPrecedingADecade.includes(lowerCaseWord)
+		const followedByFourDigitDecadePattern = fourDigitDecadeRegExp.test(nextNonWhitespaceWord)
 
+		if (isWordPrecedingAYear &&
+			(isFollowedByWhitespace || isSpecialCharacterBeforeYear) &&
+			followedByFourDigitYearPattern) {
+
+			// Normalize a four digit year pattern, e.g. 'in 1995'.
 			const normalizedString = normalizeFourDigitYearString(nextNonWhitespaceWord)
 
 			normalizedFragments.push(word)
@@ -43,11 +53,11 @@ export function getNormalizedFragmentsForSpeech(
 
 			wordIndex += 1
 		} else if (
-			wordsPrecedingADecade.includes(lowerCaseWord) &&
+			isWordPrecedingADecade &&
 			isFollowedByWhitespace &&
-			fourDigitDecadePatternRegExp.test(nextNonWhitespaceWord)) {
+			followedByFourDigitDecadePattern) {
 
- 			// Normalize a four digit decade pattern, e.g. 'the 1980s'.
+			// Normalize a four digit decade pattern, e.g. 'the 1980s'.
 
 			const normalizedString = normalizeFourDigitDecadeString(nextNonWhitespaceWord)
 
@@ -58,7 +68,7 @@ export function getNormalizedFragmentsForSpeech(
 			referenceFragments.push(nextNonWhitespaceWord)
 
 			wordIndex += 1
-		} else if (fourDigitYearRangePatternRegExp.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
+		} else if (fourDigitYearRangeRegExp.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
 			// Normalize a year range pattern, e.g. '1835-1896', ensure there are no spaces between words
 			normalizedFragments.push(normalizeFourDigitYearString(nonWhitespaceWords[wordIndex]))
 			referenceFragments.push(nonWhitespaceWords[wordIndex])
@@ -92,7 +102,7 @@ export function getNormalizedFragmentsForSpeech(
 				const referenceString = word
 				referenceFragments.push(referenceString)
 			}
-		} else if (followingCurrencyPatternRegExp.test(lowerCaseWord)) {
+		} else if (followingCurrencyRegExp.test(lowerCaseWord)) {
 			const currencyWord = currencySymbolsAsWords[currencySymbols.indexOf(lowerCaseWord[lowerCaseWord.length - 1])]
 
 			const normalizedString = `${word.substring(0, word.length - 1)} ${currencyWord}`
@@ -234,14 +244,14 @@ const wordsFollowingACurrency = [
 
 const arabicNumeralPattern = charRange('0', '9')
 
-const numberPattern = [
+const isNumberPattern = [
 	inputStart,
 	arabicNumeralPattern,
 	zeroOrMore(anyOf(arabicNumeralPattern, ',', '.')),
 	inputEnd
 ]
 
-const numberPatternRegExp = buildRegExp(numberPattern)
+const isNumberRegExp = buildRegExp(isNumberPattern)
 
 const precedingCurrencyPattern = [
 	inputStart,
@@ -261,13 +271,27 @@ const followingCurrencyPattern = [
 	inputEnd
 ]
 
-const followingCurrencyPatternRegExp = buildRegExp(followingCurrencyPattern)
+const followingCurrencyRegExp = buildRegExp(followingCurrencyPattern)
 
 const fourDigitYearPattern = [inputStart, repeated(4, arabicNumeralPattern), inputEnd]
-const fourDigitYearPatternRegExp = buildRegExp(fourDigitYearPattern)
+const fourDigitYearRegExp = buildRegExp(fourDigitYearPattern)
 
 const fourDigitDecadePattern = [inputStart, repeated(3, arabicNumeralPattern), '0s', inputEnd]
-const fourDigitDecadePatternRegExp = buildRegExp(fourDigitDecadePattern)
+const fourDigitDecadeRegExp = buildRegExp(fourDigitDecadePattern)
 
 const fourDigitYearRangePattern = [inputStart, repeated(4, arabicNumeralPattern), anyOf('-', '–'), repeated(4, arabicNumeralPattern), inputEnd]
-const fourDigitYearRangePatternRegExp = buildRegExp(fourDigitYearRangePattern)
+const fourDigitYearRangeRegExp = buildRegExp(fourDigitYearRangePattern)
+
+const isAllLettersPattern = [inputStart, oneOrMore(unicodeProperty('Letter')), inputEnd]
+const isAllLettersRegExp = buildRegExp(isAllLettersPattern)
+
+const isAllLettersOrApostrophePattern = [
+	inputStart,
+
+	unicodeProperty('Letter'),
+	zeroOrMore(anyOf(unicodeProperty('Letter'), `'`)),
+
+	inputEnd,
+]
+
+const isAllLettersOrApostropheRegExp = buildRegExp(isAllLettersOrApostrophePattern)

package/src/recognition/AmazonTranscribeSTT.ts

@@ -1,6 +1,6 @@
-import { Item, LanguageCode, StartStreamTranscriptionCommandInput } from '@aws-sdk/client-transcribe-streaming'
-import { wordCharacterRegExp } from '../nlp/Segmentation.js'
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Item, LanguageCode, StartStreamTranscriptionCommandInput } from '@aws-sdk/client-transcribe-streaming'
+import { isWordOrEmojiOrSymbolWord } from '../nlp/Segmentation.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
@@ -103,7 +103,7 @@ export async function recgonize(rawAudio: RawAudio, languageCode: string, region
 	for (const event of events) {
 		const text = event.Content!
 
-		if (!wordCharacterRegExp.test(text)) {
+		if (!isWordOrEmojiOrSymbolWord(text)) {
 			continue
 		}
 

package/src/synthesis/AzureCognitiveServicesTTS.ts

@@ -6,7 +6,7 @@ import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { RawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
 import { concatUint8Arrays } from '../utilities/Utilities.js'
-import { escapeHtml } from '../encodings/HtmlEscape.js'
+import { escapeHtml } from 'standard-html-escaper'
 import { SynthesisCallbacks } from '../api/Synthesis.js'
 
 export async function synthesize(

package/src/synthesis/EspeakTTS.ts

@@ -4,15 +4,15 @@ import { Logger } from '../utilities/Logger.js'
 import { RawAudio, getEmptyRawAudio } from '../audio/AudioUtilities.js'
 import { getNormalizedFragmentsForSpeech, simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
 import { ipaPhoneToKirshenbaum } from '../nlp/PhoneConversion.js'
-import { isAllPunctuation, isWord, isWordOrEmoji, splitToWords, wordCharacterRegExp } from '../nlp/Segmentation.js'
+import { isAllPunctuation, isWordOrEmoji, splitToWords, isWordOrEmojiOrSymbolWord } from '../nlp/Segmentation.js'
 import { Lexicon, tryGetFirstLexiconSubstitution } from '../nlp/Lexicon.js'
 import { phonemizeSentence } from '../nlp/EspeakPhonemizer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
-import { escapeHtml } from '../encodings/HtmlEscape.js'
+import { escapeHtml } from 'standard-html-escaper'
 import * as TextSegmentation from '@echogarden/text-segmentation'
 
-import { getGlobalOption, OperationCallbacks, SynthesisCallbacks } from '../api/API.js'
+import { OperationCallbacks, SynthesisCallbacks } from '../api/API.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 
 import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
@@ -73,11 +73,14 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 		for (let i = 0; i < mergedWords.length; i++) {
 			const mergedWord = mergedWords[i]
 
-			// Convert isolated groups of vertical bars and em dashes to a comma
 			if (/^[\|│—─–]+$/.test(mergedWord)) {
+				// Convert isolated groups of vertical bars or em dashes to a comma
 				mergedWords[i] = ','
-			} else if (isAllPunctuation(mergedWord)) { // Collapse repeated punctuation
-				mergedWords[i] = mergedWord[0]
+			} else if (isAllPunctuation(mergedWord)) {
+				// Collapse repeated punctuation to up to 3 repetitions,
+				// Since otherwise eSpeak may go crazy.
+
+				mergedWords[i] = mergedWord.substring(0, Math.min(3, mergedWord.length))
 			}
 		}
 
@@ -102,7 +105,12 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 
 	const { normalizedFragments, referenceFragments } = getNormalizedFragmentsForSpeech(words, nonWhitespaceWords, nonWhitespaceWordsOriginalIndex, language)
 
-	const simplifiedFragments = normalizedFragments.map(word => simplifyPunctuationCharacters(word).toLocaleLowerCase())
+	const simplifiedFragments = normalizedFragments.map(word => {
+		return simplifyPunctuationCharacters(word)
+			.toLocaleLowerCase()
+			.replaceAll('(', ',')
+			.replaceAll(')', ',')
+	})
 
 	if ([`'`].includes(simplifiedFragments[0])) {
 		normalizedFragments[0] = `()`
@@ -150,8 +158,8 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 	{
 		const fragmentWordSequence = new TextSegmentation.WordSequence()
 
-		for (let fragment of fragments) {
-			fragmentWordSequence.addWord(fragment, 0, !wordCharacterRegExp.test(fragment))
+		for (let fragment of simplifiedFragments) {
+			fragmentWordSequence.addWord(fragment, 0, !isWordOrEmojiOrSymbolWord(fragment))
 		}
 
 		const wordEntries = referenceTimeline.flatMap(phraseEntry => phraseEntry.timeline!)

package/src/synthesis/KokoroTTS.ts

@@ -72,12 +72,6 @@ export class KokoroTTS {
 		const voicePrimaryLanguage = voice.languages[0]
 		const voicePrimaryLanguageShort = getShortLanguageCode(voicePrimaryLanguage)
 
-		sentenceText = //simplifyPunctuationCharacters(sentence.trim())
-			sentenceText
-				.replaceAll('(', ', ')
-				.replaceAll(')', ', ')
-				.replaceAll('—', ', ')
-
 		const simplifiedSentenceText = simplifyPunctuationCharacters(sentenceText.trim())
 
 		const voiceLanguage = voiceEntry.languages[0]

package/src/synthesis/MicrosoftEdgeTTS.ts

@@ -8,7 +8,7 @@ import { concatUint8Arrays, getRandomHexString, logToStderr, sha256AsHex } from
 import { RawAudio, getEmptyRawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
 import { Timer } from '../utilities/Timer.js'
 import { decodeUtf8, encodeUtf8 } from '../encodings/Utf8.js'
-import { escapeHtml } from '../encodings/HtmlEscape.js'
+import { escapeHtml } from 'standard-html-escaper'
 import { requestHttp } from 'easier-http-request'
 import { VoiceListRequestCallbacks, SynthesisCallbacks } from '../api/API.js'
 

package/src/synthesis/VitsTTS.ts

@@ -72,12 +72,6 @@ export class VitsTTS {
 
 		lengthScale *= baseLengthScale
 
-		sentence = //simplifyPunctuationCharacters(sentence.trim())
-			sentence
-				.replaceAll('(', ', ')
-				.replaceAll(')', ', ')
-				.replaceAll('—', ', ')
-
 		const Espeak = await import('../synthesis/EspeakTTS.js')
 
 		logger.end()