echogarden 3.0.4 → 3.1.0

package/src/api/Alignment.ts

@@ -14,7 +14,7 @@ import { type WhisperAlignmentOptions } from '../recognition/WhisperSTT.js'
 import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
 import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 import { type EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
-import { isWord } from '../nlp/Segmentation.js'
+import { includesWordCharacter } from '../nlp/Segmentation.js'
 
 const log = logToStderr
 
@@ -264,7 +264,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			logger.log('')
 
 			// Remove non-word entries from recognition timeline
-			recognitionTimeline = recognitionTimeline.filter(entry => isWord(entry.text))
+			recognitionTimeline = recognitionTimeline.filter(entry => includesWordCharacter(entry.text))
 
 			// Synthesize the ground-truth transcript and get its timeline
 			logger.start('Synthesize ground-truth transcript with eSpeak')

package/src/nlp/Segmentation.ts

@@ -1,7 +1,8 @@
 import { sumArray, logToStderr } from '../utilities/Utilities.js'
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
-import { includesAnyOf, splitAndPreserveSeparators } from '../utilities/StringUtilities.js'
+import { splitAndPreserveSeparators } from '../utilities/StringUtilities.js'
+import { anyOf, buildRegExp, unicodeProperty, oneOrMore, possibly, codepoint, inputStart, inputEnd, whitespace } from 'regexp-composer'
 
 import * as TextSegmentation from '@echogarden/text-segmentation'
 import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js'
@@ -9,50 +10,77 @@ import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js'
 
 const log = logToStderr
 
-export const wordCharacterRegExp = /[\p{Letter}\p{Number}]/u
+const includesWordCharacterPattern = anyOf(unicodeProperty('Letter'), unicodeProperty('Number'))
+const includesWordCharacterRegExp = buildRegExp(includesWordCharacterPattern)
 
 // See: https://mathiasbynens.be/notes/es-unicode-property-escapes
-export const emojiSequenceRegExp = /\p{Emoji_Modifier_Base}\p{Emoji_Modifier}?|\p{Emoji_Presentation}|\p{Emoji}\uFE0F/u
+//export const emojiSequenceRegExp = /\p{Emoji_Modifier_Base}\p{Emoji_Modifier}?|\p{Emoji_Presentation}|\p{Emoji}\uFE0F/u
+const includesEmojiSequencePattern = anyOf(
+	[unicodeProperty('Emoji_Modifier_Base'), possibly(unicodeProperty('Emoji_Modifier'))],
+	unicodeProperty('Emoji_Presentation'),
+	[unicodeProperty('Emoji'), codepoint('FE0F')]
+)
+const includesEmojiSequenceRegExp = buildRegExp(includesEmojiSequencePattern)
 
-export const includesPunctuationRegExp = /[\p{Punctuation}]/u
-export const isAllPunctuationRegExp = /^[\p{Punctuation}]+$/u
+const symbolWordsList = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '\\', '^', '*', '×', '=', '≈', '¼', '½', '¾', '→', '≤', '≥']
+const includesSymbolWordPattern = anyOf(...symbolWordsList)
+const includesSymbolWordRegExp = buildRegExp(includesSymbolWordPattern)
 
-export const phraseSeparators = [',', '、', '，', '،', ';', '；', ':', '：', '—']
-export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '\\', '^', '*', '×', '=', '≈', '¼', '½', '¾', '→', '≤', '≥']
+const isAllSymbolWordsPattern = [inputStart, oneOrMore(includesSymbolWordPattern), inputEnd]
+const isAllSymbolWordsRegExp = buildRegExp(isAllSymbolWordsPattern)
+
+const includesWordCharacterOrEmojiPattern = anyOf(includesWordCharacterPattern, includesEmojiSequencePattern)
+const includesWordCharacterOrEmojiRegExp = buildRegExp(includesWordCharacterOrEmojiPattern)
+
+const includesWordCharacterOrEmojiOrIsAllSymbolWordPattern = anyOf(includesWordCharacterPattern, includesEmojiSequencePattern, isAllSymbolWordsPattern)
+const includesWordCharacterOrEmojiOrIsAllSymbolWordRegExp = buildRegExp(includesWordCharacterOrEmojiOrIsAllSymbolWordPattern)
+
+const includesPunctuationPattern = unicodeProperty('Punctuation')
+const includesPunctuationRegExp = buildRegExp(includesPunctuationPattern)
+
+const isAllPunctuationPattern = [inputStart, oneOrMore(includesPunctuationPattern), inputEnd]
+const isAllPunctuationRegExp = buildRegExp(isAllPunctuationPattern)
+
+const phraseSeparatorsList = [',', '、', '，', '،', ';', '；', ':', '：', '—']
+const includesPhraseSeparatorsPattern = anyOf(...phraseSeparatorsList)
+const includesPhraseSeparatorsRegExp = buildRegExp(includesPhraseSeparatorsPattern)
+
+const isAllWhitespacePattern = [inputStart, oneOrMore(whitespace), inputEnd]
+const isAllWhitespaceRegExp = buildRegExp(isAllWhitespacePattern)
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
 // Predicates
 ///////////////////////////////////////////////////////////////////////////////////////////////
 export function isWordOrEmojiOrSymbolWord(str: string) {
-	return isWordOrEmoji(str) || symbolWords.includes(str)
+	return includesWordCharacterOrEmojiOrIsAllSymbolWordRegExp.test(str.trim())
 }
 
 export function isWordOrEmoji(str: string) {
-	return isWord(str) || includesEmoji(str)
+	return includesWordCharacterOrEmojiRegExp.test(str.trim())
 }
 
 export function isSymbolWord(str: string) {
-	return symbolWords.includes(str?.trim())
+	return isAllSymbolWordsRegExp.test(str.trim())
 }
 
-export function isWord(str: string) {
-	return wordCharacterRegExp.test(str?.trim())
+export function includesWordCharacter(str: string) {
+	return includesWordCharacterRegExp.test(str.trim())
 }
 
 export function includesPunctuation(str: string) {
-	return includesPunctuationRegExp.test(str?.trim())
+	return includesPunctuationRegExp.test(str.trim())
 }
 
 export function isAllPunctuation(str: string) {
-	return isAllPunctuationRegExp.test(str?.trim())
+	return isAllPunctuationRegExp.test(str.trim())
 }
 
 export function includesEmoji(str: string) {
-	return emojiSequenceRegExp.test(str?.trim())
+	return includesEmojiSequenceRegExp.test(str.trim())
 }
 
 export function isAllWhitespace(str: string) {
-	return str && /^\s+$/.test(str)
+	return isAllWhitespaceRegExp.test(str)
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
@@ -280,11 +308,13 @@ export class Word {
 		this.isSentenceFinalizer = isSentenceFinalizer
 	}
 
-	get containsOnlyPunctuation() { return !wordCharacterRegExp.test(this.text) && !this.isSymbolWord }
+	get containsOnlyPunctuation() { return !isWordOrEmojiOrSymbolWord(this.text) }
 
-	get isSymbolWord() { return symbolWords.includes(this.text) }
+	get isSymbolWord() { return isSymbolWord(this.text) }
 
-	get isPhraseSeperator() { return this.containsOnlyPunctuation && includesAnyOf(this.text, phraseSeparators) }
+	get isPhraseSeperator() {
+		return this.containsOnlyPunctuation && includesPhraseSeparatorsRegExp.test(this.text)
+	}
 
 	get length() { return this.text.length }
 }

package/src/nlp/TextNormalizer.ts

@@ -1,6 +1,6 @@
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { substituteCharactersUsingLookup } from '../utilities/StringUtilities.js'
-import { anyOf, buildRegExp, charRange, inputEnd, inputStart, repeated, zeroOrMore } from 'regexp-composer'
+import { anyOf, buildRegExp, charRange, inputEnd, inputStart, oneOrMore, repeated, unicodeProperty, zeroOrMore } from 'regexp-composer'
 
 export function getNormalizedFragmentsForSpeech(
 	words: string[],
@@ -25,14 +25,24 @@ export function getNormalizedFragmentsForSpeech(
 		const nextNonWhitespaceWord = nextNonWhitespaceWords[0]
 
 		const originalWordIndex = nonWhitespaceWordOriginalIndex[wordIndex]
+
 		const isFollowedByWhitespace = words[originalWordIndex + 1]?.trim().length === 0
+		const isSpecialCharacterBeforeYear = ['(', ',', '©'].includes(words[originalWordIndex])
 
-		if (wordsPrecedingAYear.includes(lowerCaseWord) &&
-			isFollowedByWhitespace &&
-			fourDigitYearPatternRegExp.test(nextNonWhitespaceWord)) {
+		//const isWordPrecedingAYear = wordsPrecedingAYear.includes(lowerCaseWord)
+		const isWordPrecedingAYear =
+			isAllLettersRegExp.test(lowerCaseWord) || isSpecialCharacterBeforeYear
+
+		const followedByFourDigitYearPattern = fourDigitYearRegExp.test(nextNonWhitespaceWord)
 
- 			// Normalize a four digit year pattern, e.g. 'in 1995'.
+		const isWordPrecedingADecade = wordsPrecedingADecade.includes(lowerCaseWord)
+		const followedByFourDigitDecadePattern = fourDigitDecadeRegExp.test(nextNonWhitespaceWord)
 
+		if (isWordPrecedingAYear &&
+			(isFollowedByWhitespace || isSpecialCharacterBeforeYear) &&
+			followedByFourDigitYearPattern) {
+
+			// Normalize a four digit year pattern, e.g. 'in 1995'.
 			const normalizedString = normalizeFourDigitYearString(nextNonWhitespaceWord)
 
 			normalizedFragments.push(word)
@@ -43,11 +53,11 @@ export function getNormalizedFragmentsForSpeech(
 
 			wordIndex += 1
 		} else if (
-			wordsPrecedingADecade.includes(lowerCaseWord) &&
+			isWordPrecedingADecade &&
 			isFollowedByWhitespace &&
-			fourDigitDecadePatternRegExp.test(nextNonWhitespaceWord)) {
+			followedByFourDigitDecadePattern) {
 
- 			// Normalize a four digit decade pattern, e.g. 'the 1980s'.
+			// Normalize a four digit decade pattern, e.g. 'the 1980s'.
 
 			const normalizedString = normalizeFourDigitDecadeString(nextNonWhitespaceWord)
 
@@ -58,7 +68,7 @@ export function getNormalizedFragmentsForSpeech(
 			referenceFragments.push(nextNonWhitespaceWord)
 
 			wordIndex += 1
-		} else if (fourDigitYearRangePatternRegExp.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
+		} else if (fourDigitYearRangeRegExp.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
 			// Normalize a year range pattern, e.g. '1835-1896', ensure there are no spaces between words
 			normalizedFragments.push(normalizeFourDigitYearString(nonWhitespaceWords[wordIndex]))
 			referenceFragments.push(nonWhitespaceWords[wordIndex])
@@ -92,7 +102,7 @@ export function getNormalizedFragmentsForSpeech(
 				const referenceString = word
 				referenceFragments.push(referenceString)
 			}
-		} else if (followingCurrencyPatternRegExp.test(lowerCaseWord)) {
+		} else if (followingCurrencyRegExp.test(lowerCaseWord)) {
 			const currencyWord = currencySymbolsAsWords[currencySymbols.indexOf(lowerCaseWord[lowerCaseWord.length - 1])]
 
 			const normalizedString = `${word.substring(0, word.length - 1)} ${currencyWord}`
@@ -234,14 +244,14 @@ const wordsFollowingACurrency = [
 
 const arabicNumeralPattern = charRange('0', '9')
 
-const numberPattern = [
+const isNumberPattern = [
 	inputStart,
 	arabicNumeralPattern,
 	zeroOrMore(anyOf(arabicNumeralPattern, ',', '.')),
 	inputEnd
 ]
 
-const numberPatternRegExp = buildRegExp(numberPattern)
+const isNumberRegExp = buildRegExp(isNumberPattern)
 
 const precedingCurrencyPattern = [
 	inputStart,
@@ -261,13 +271,16 @@ const followingCurrencyPattern = [
 	inputEnd
 ]
 
-const followingCurrencyPatternRegExp = buildRegExp(followingCurrencyPattern)
+const followingCurrencyRegExp = buildRegExp(followingCurrencyPattern)
 
 const fourDigitYearPattern = [inputStart, repeated(4, arabicNumeralPattern), inputEnd]
-const fourDigitYearPatternRegExp = buildRegExp(fourDigitYearPattern)
+const fourDigitYearRegExp = buildRegExp(fourDigitYearPattern)
 
 const fourDigitDecadePattern = [inputStart, repeated(3, arabicNumeralPattern), '0s', inputEnd]
-const fourDigitDecadePatternRegExp = buildRegExp(fourDigitDecadePattern)
+const fourDigitDecadeRegExp = buildRegExp(fourDigitDecadePattern)
 
 const fourDigitYearRangePattern = [inputStart, repeated(4, arabicNumeralPattern), anyOf('-', '–'), repeated(4, arabicNumeralPattern), inputEnd]
-const fourDigitYearRangePatternRegExp = buildRegExp(fourDigitYearRangePattern)
+const fourDigitYearRangeRegExp = buildRegExp(fourDigitYearRangePattern)
+
+const isAllLettersPattern = [inputStart, oneOrMore(unicodeProperty('Letter')), inputEnd]
+const isAllLettersRegExp = buildRegExp(isAllLettersPattern)

package/src/recognition/AmazonTranscribeSTT.ts

@@ -1,6 +1,6 @@
-import { Item, LanguageCode, StartStreamTranscriptionCommandInput } from '@aws-sdk/client-transcribe-streaming'
-import { wordCharacterRegExp } from '../nlp/Segmentation.js'
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Item, LanguageCode, StartStreamTranscriptionCommandInput } from '@aws-sdk/client-transcribe-streaming'
+import { isWordOrEmojiOrSymbolWord } from '../nlp/Segmentation.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
@@ -103,7 +103,7 @@ export async function recgonize(rawAudio: RawAudio, languageCode: string, region
 	for (const event of events) {
 		const text = event.Content!
 
-		if (!wordCharacterRegExp.test(text)) {
+		if (!isWordOrEmojiOrSymbolWord(text)) {
 			continue
 		}
 

package/src/synthesis/EspeakTTS.ts

@@ -4,7 +4,7 @@ import { Logger } from '../utilities/Logger.js'
 import { RawAudio, getEmptyRawAudio } from '../audio/AudioUtilities.js'
 import { getNormalizedFragmentsForSpeech, simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
 import { ipaPhoneToKirshenbaum } from '../nlp/PhoneConversion.js'
-import { isAllPunctuation, isWord, isWordOrEmoji, splitToWords, wordCharacterRegExp } from '../nlp/Segmentation.js'
+import { isAllPunctuation, isWordOrEmoji, splitToWords, isWordOrEmojiOrSymbolWord } from '../nlp/Segmentation.js'
 import { Lexicon, tryGetFirstLexiconSubstitution } from '../nlp/Lexicon.js'
 import { phonemizeSentence } from '../nlp/EspeakPhonemizer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
@@ -12,7 +12,7 @@ import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { escapeHtml } from '../encodings/HtmlEscape.js'
 import * as TextSegmentation from '@echogarden/text-segmentation'
 
-import { getGlobalOption, OperationCallbacks, SynthesisCallbacks } from '../api/API.js'
+import { OperationCallbacks, SynthesisCallbacks } from '../api/API.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 
 import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
@@ -73,11 +73,14 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 		for (let i = 0; i < mergedWords.length; i++) {
 			const mergedWord = mergedWords[i]
 
-			// Convert isolated groups of vertical bars and em dashes to a comma
 			if (/^[\|│—─–]+$/.test(mergedWord)) {
+				// Convert isolated groups of vertical bars or em dashes to a comma
 				mergedWords[i] = ','
-			} else if (isAllPunctuation(mergedWord)) { // Collapse repeated punctuation
-				mergedWords[i] = mergedWord[0]
+			} else if (isAllPunctuation(mergedWord)) {
+				// Collapse repeated punctuation to up to 3 repetitions,
+				// Since otherwise eSpeak may go crazy.
+
+				mergedWords[i] = mergedWord.substring(0, Math.min(3, mergedWord.length))
 			}
 		}
 
@@ -102,7 +105,12 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 
 	const { normalizedFragments, referenceFragments } = getNormalizedFragmentsForSpeech(words, nonWhitespaceWords, nonWhitespaceWordsOriginalIndex, language)
 
-	const simplifiedFragments = normalizedFragments.map(word => simplifyPunctuationCharacters(word).toLocaleLowerCase())
+	const simplifiedFragments = normalizedFragments.map(word => {
+		return simplifyPunctuationCharacters(word)
+			.toLocaleLowerCase()
+			.replaceAll('(', ',')
+			.replaceAll(')', ',')
+	})
 
 	if ([`'`].includes(simplifiedFragments[0])) {
 		normalizedFragments[0] = `()`
@@ -150,8 +158,8 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 	{
 		const fragmentWordSequence = new TextSegmentation.WordSequence()
 
-		for (let fragment of fragments) {
-			fragmentWordSequence.addWord(fragment, 0, !wordCharacterRegExp.test(fragment))
+		for (let fragment of simplifiedFragments) {
+			fragmentWordSequence.addWord(fragment, 0, !isWordOrEmojiOrSymbolWord(fragment))
 		}
 
 		const wordEntries = referenceTimeline.flatMap(phraseEntry => phraseEntry.timeline!)

package/src/synthesis/KokoroTTS.ts

@@ -72,12 +72,6 @@ export class KokoroTTS {
 		const voicePrimaryLanguage = voice.languages[0]
 		const voicePrimaryLanguageShort = getShortLanguageCode(voicePrimaryLanguage)
 
-		sentenceText = //simplifyPunctuationCharacters(sentence.trim())
-			sentenceText
-				.replaceAll('(', ', ')
-				.replaceAll(')', ', ')
-				.replaceAll('—', ', ')
-
 		const simplifiedSentenceText = simplifyPunctuationCharacters(sentenceText.trim())
 
 		const voiceLanguage = voiceEntry.languages[0]

package/src/synthesis/VitsTTS.ts

@@ -72,12 +72,6 @@ export class VitsTTS {
 
 		lengthScale *= baseLengthScale
 
-		sentence = //simplifyPunctuationCharacters(sentence.trim())
-			sentence
-				.replaceAll('(', ', ')
-				.replaceAll(')', ', ')
-				.replaceAll('—', ', ')
-
 		const Espeak = await import('../synthesis/EspeakTTS.js')
 
 		logger.end()

package/src/utilities/Timeline.ts

@@ -1,6 +1,6 @@
 import { addMissingPunctuationWordsToWordSequence, segmentWordSequence, WordSequence } from '@echogarden/text-segmentation'
 import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
-import { applyWhitespaceProcessing, isWord, isWordOrEmojiOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js'
+import { applyWhitespaceProcessing, includesWordCharacter, isWordOrEmojiOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js'
 import { deepClone } from './ObjectUtilities.js'
 import { getUTF32Chars } from './StringUtilities.js'
 import { roundToDigits } from './Utilities.js'
@@ -213,7 +213,7 @@ function replaceSentenceEndersWithinWordsWithMaskingCharacter(transcript: string
 	for (const wordEntry of wordTimeline) {
 		const wordText = wordEntry.text
 
-		if (!isWord(wordText)) {
+		if (!includesWordCharacter(wordText)) {
 			continue
 		}
 

package/src/utilities/WikipediaReader.ts

@@ -1,10 +1,10 @@
 import { OperationCallbacks } from '../api/Common.js'
-import { splitToParagraphs, wordCharacterRegExp } from '../nlp/Segmentation.js'
+import { isWordOrEmojiOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js'
 
 export async function fetchAndParseWikipediaArticle(articleName: string, language: string, callbacks: OperationCallbacks) {
-	const { default: wtf } = await import('wtf_wikipedia')
+	const { default: wtfWikipedia } = await import('wtf_wikipedia')
 
-	const document = await wtf.fetch(articleName, language)
+	const document = await wtfWikipedia.fetch(articleName, language)
 
 	if (!document) {
 		throw new Error('Error fetching Wikipedia article')
@@ -16,7 +16,7 @@ export async function fetchAndParseWikipediaArticle(articleName: string, languag
 	for (const section of sections) {
 		const sectionTitle = section.title()
 
-		if (wordCharacterRegExp.test(sectionTitle)) {
+		if (isWordOrEmojiOrSymbolWord(sectionTitle)) {
 			sectionsText.push(sectionTitle)
 		}
 
@@ -25,7 +25,7 @@ export async function fetchAndParseWikipediaArticle(articleName: string, languag
 		for (const paragraph of sectionParagraphs) {
 			const paragraphText = paragraph
 
-			if (wordCharacterRegExp.test(paragraphText)) {
+			if (isWordOrEmojiOrSymbolWord(paragraphText)) {
 				sectionsText.push(paragraphText)
 			}
 		}