echogarden 2.7.0 → 2.8.0

package/src/nlp/Segmentation.ts

@@ -1,22 +1,26 @@
-import * as CldrSegmentation from 'cldr-segmentation'
-import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js'
-
 import { sumArray, logToStderr } from '../utilities/Utilities.js'
 import { getShortLanguageCode } from '../utilities/Locale.js'
-import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js'
 import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
-import { includesAnyOf, indexOfAnyOf } from '../utilities/StringUtilities.js'
+import { includesAnyOf, splitAndPreserveSeparators } from '../utilities/StringUtilities.js'
+
+import * as TextSegmentation from '@echogarden/text-segmentation'
+import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js'
+import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js'
 
 const log = logToStderr
 
-export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u
-export const punctuationPattern = /[\p{Punctuation}]/u
+export const wordCharacterRegExp = /[\p{Letter}\p{Number}]/u
+export const emojiCharacterRegExp = /[\p{Emoji}]/u
+export const punctuationRegExp = /[\p{Punctuation}]/u
 
-export const phraseSeparators = [',', ';', ':', '，', '、']
+export const phraseSeparators = [',', '、', '，', '،', ';', '；', ':', '：', '—']
 export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '=', '¼', '½', '¾']
 
+///////////////////////////////////////////////////////////////////////////////////////////////
+// Predicates
+///////////////////////////////////////////////////////////////////////////////////////////////
 export function isWordOrSymbolWord(str: string) {
-	return isWord(str) || symbolWords.includes(str)
+	return isWord(str) || includesEmoji(str) || symbolWords.includes(str)
 }
 
 export function isSymbolWord(str: string) {
@@ -24,87 +28,106 @@ export function isSymbolWord(str: string) {
 }
 
 export function isWord(str: string) {
-	return wordCharacterPattern.test(str.trim())
+	return wordCharacterRegExp.test(str.trim())
+}
+
+export function includesPunctuation(str: string) {
+	return punctuationRegExp.test(str.trim())
 }
 
-export function isPunctuation(str: string) {
-	return punctuationPattern.test(str.trim())
+export function includesEmoji(str: string) {
+	return emojiCharacterRegExp.test(str.trim())
 }
 
 export function isWhitespace(str: string) {
 	return str.trim().length === 0
 }
 
-export class Sentence {
-	phrases: Phrase[] = []
+///////////////////////////////////////////////////////////////////////////////////////////////
+// Paragraph, line, sentence, phrase, and word segmentation
+///////////////////////////////////////////////////////////////////////////////////////////////
+export function splitToParagraphs(text: string, paragraphBreakType: ParagraphBreakType, whitespaceProcessingMethod: WhitespaceProcessing) {
+	let paragraphs: string[] = []
 
-	readonly isSentenceFinalizer = true
+	if (paragraphBreakType === 'single') {
+		paragraphs = splitAndPreserveSeparators(text, /(\r?\n)+/g)
+	} else if (paragraphBreakType === 'double') {
+		paragraphs = splitAndPreserveSeparators(text, /(\r?\n)(\r?\n)+/g)
+	} else {
+		throw new Error(`Invalid paragraph break type: '${paragraphBreakType}'`)
+	}
 
-	get length() { return sumArray(this.phrases, (phrase) => phrase.length) }
+	paragraphs = paragraphs.map(p => applyWhitespaceProcessing(p, whitespaceProcessingMethod))
+	paragraphs = paragraphs.filter(p => p.length > 0)
 
-	get text() { return this.phrases.reduce<string>((result, phrase) => result + phrase.text, '') }
+	return paragraphs
 }
 
-export class Phrase {
-	words: Word[] = []
-
-	get length() { return sumArray(this.words, (word) => word.length) }
-
-	get text() { return this.words.reduce<string>((result, word) => result + word.text, '') }
-
-	get lastWord() {
-		if (this.words.length == 0) {
-			return undefined
-		}
-
-		return this.words[this.words.length - 1]
-	}
-
-	get isSentenceFinalizer() { return this.lastWord != null ? this.lastWord.isSentenceFinalizer : false }
+export function splitToLines(text: string) {
+	return splitAndPreserveSeparators(text, /\r?\n/g)
 }
 
-export class Word {
-	readonly text: string
-	isSentenceFinalizer: boolean
-
-	constructor(text: string, isSentenceFinalizer: boolean) {
-		this.text = text
-		this.isSentenceFinalizer = isSentenceFinalizer
-	}
-
-	get containsOnlyPunctuation() { return !wordCharacterPattern.test(this.text) && !this.isSymbolWord }
+export async function parseText(text: string, langCode: string) {
+	const shortLangCode = getShortLanguageCode(langCode || '')
 
-	get isSymbolWord() { return symbolWords.includes(this.text) }
+	const wordSequence = await splitToWords(text, shortLangCode)
 
-	get isPhraseSeperator() { return this.containsOnlyPunctuation && includesAnyOf(this.text, phraseSeparators) }
+	const parsedText = await TextSegmentation.segmentWordSequence(wordSequence)
 
-	get length() { return this.text.length }
+	return parsedText
 }
 
-export type Segment = Sentence | Phrase | Word
+export async function splitToWords(text: string, langCode: string): Promise<TextSegmentation.WordSequence> {
+	const shortLangCode = getShortLanguageCode(langCode || '')
 
-export class Fragment {
-	segments: Segment[] = []
+	if (shortLangCode === 'zh' || shortLangCode === 'cmn' || shortLangCode === 'ja') {
+		let wordArray: string[] = []
 
-	get length() { return sumArray(this.segments, (phrase) => phrase.length) }
+		if (shortLangCode === 'zh' || shortLangCode === 'cmn') {
+			wordArray = await splitChineseTextToWords_Jieba(text)
+		} else {
+			wordArray = await splitJapaneseTextToWords_Kuromoji(text)
+		}
 
-	get text() { return this.segments.reduce<string>((result, segment) => result + segment.text, '') }
+		const wordSequence = new TextSegmentation.WordSequence()
 
-	get isEmpty() { return this.length == 0 }
+		let offset = 0
 
-	get isNonempty() { return !this.isEmpty }
+		for (const wordText of wordArray) {
+			const startOffset = offset
+			const endOffset = startOffset + wordText.length
+			const isNonPunctuation = isWordOrSymbolWord(wordText)
 
-	get lastSegment() {
-		if (this.isEmpty) {
-			return undefined
+			wordSequence.addWord(wordText, startOffset, isNonPunctuation)
+
+			offset = endOffset
 		}
 
-		return this.segments[this.segments.length - 1]
+		return wordSequence
+	} else {
+		return TextSegmentation.splitToWords(text, { language: langCode })
+	}
+}
+
+export function applyWhitespaceProcessing(text: string, whitespaceProcessingMethod: WhitespaceProcessing) {
+	if (whitespaceProcessingMethod === 'removeLineBreaks') {
+		return text.trim().replaceAll(/(\r?\n)+/g, ' ')
+	} else if (whitespaceProcessingMethod === 'collapse') {
+		return text.trim().replaceAll(/\s+/g, ' ')
+	} else if (whitespaceProcessingMethod === 'preserve') {
+		return text
+	} else {
+		throw new Error(`Invalid whitespace processing method: '${whitespaceProcessingMethod}'`)
 	}
 }
 
+///////////////////////////////////////////////////////////////////////////////////////////////
+// Fragment segmentation
+//
+// Used to split text to fragments, to fit particular size constraints.
+///////////////////////////////////////////////////////////////////////////////////////////////
 export async function splitToFragments(text: string, maxFragmentLength: number, langCode: string, preserveSentences = true, preservePhrases = true) {
-	const parsedText = await parse(text, langCode)
+	const parsedText = await parseTextAndConvertToFragmentObjects(text, langCode)
 
 	const fragments: Fragment[] = []
 	let currentFragment = new Fragment()
@@ -173,33 +196,28 @@ export async function splitToFragments(text: string, maxFragmentLength: number,
 	return fragments
 }
 
-export async function parse(text: string, langCode: string) {
-	const sentencesText = splitToSentences(text, langCode)
+export async function parseTextAndConvertToFragmentObjects(text: string, langCode: string) {
+	const segmentedText = await parseText(text, langCode)
+
 	const sentences: Sentence[] = []
 
-	for (const sentenceText of sentencesText) {
+	for (const sentenceEntry of segmentedText.sentences) {
 		const sentence = new Sentence()
 
-		let currentPhrase = new Phrase()
-		const wordTexts = await splitToWords(sentenceText, langCode)
+		for (const phraseEntry of sentenceEntry.phrases) {
+			const phrase = new Phrase()
 
-		for (let wordIndex = 0; wordIndex < wordTexts.length; wordIndex++) {
-			const word = new Word(wordTexts[wordIndex], wordIndex == wordTexts.length - 1)
+			for (const wordEntry of phraseEntry.words.entries) {
+				const isSentenceFinalizer = wordEntry === sentenceEntry.words.lastEntry
 
-			if (word.isPhraseSeperator) {
-				const separatorIndex = indexOfAnyOf(word.text, phraseSeparators)
-				currentPhrase.words.push(new Word(word.text.substring(0, separatorIndex + 1), word.isSentenceFinalizer))
-				sentence.phrases.push(currentPhrase)
+				const word = new Word(wordEntry.text, isSentenceFinalizer)
 
-				currentPhrase = new Phrase()
-				currentPhrase.words.push(new Word(word.text.substring(separatorIndex + 1), false))
-			} else {
-				currentPhrase.words.push(word)
+				phrase.words.push(word)
 			}
-		}
 
-		if (currentPhrase.words.length > 0) {
-			sentence.phrases.push(currentPhrase)
+			if (phrase.words.length > 0) {
+				sentence.phrases.push(phrase)
+			}
 		}
 
 		sentences.push(sentence)
@@ -208,53 +226,70 @@ export async function parse(text: string, langCode: string) {
 	return sentences
 }
 
-export function splitToSentences(text: string, langCode: string): string[] {
-	const shortLangCode = getShortLanguageCode(langCode || '')
+export class Sentence {
+	phrases: Phrase[] = []
+
+	readonly isSentenceFinalizer = true
 
-	return CldrSegmentation.sentenceSplit(text, CldrSegmentation.suppressions[shortLangCode])
+	get length() { return sumArray(this.phrases, (phrase) => phrase.length) }
+
+	get text() { return this.phrases.reduce<string>((result, phrase) => result + phrase.text, '') }
 }
 
-export async function splitToWords(text: string, langCode: string): Promise<string[]> {
-	const shortLangCode = getShortLanguageCode(langCode || '')
+export class Phrase {
+	words: Word[] = []
 
-	if (shortLangCode == 'zh' || shortLangCode == 'cmn') {
-		return splitChineseTextToWords_Jieba(text, undefined, true)
-	} else if (shortLangCode == 'ja') {
-		return splitJapaneseTextToWords_Kuromoji(text)
-	} else {
-		return CldrSegmentation.wordSplit(text, CldrSegmentation.suppressions[shortLangCode])
+	get length() { return sumArray(this.words, (word) => word.length) }
+
+	get text() { return this.words.reduce<string>((result, word) => result + word.text, '') }
+
+	get lastWord() {
+		if (this.words.length == 0) {
+			return undefined
+		}
+
+		return this.words[this.words.length - 1]
 	}
+
+	get isSentenceFinalizer() { return this.lastWord != null ? this.lastWord.isSentenceFinalizer : false }
 }
 
-export function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakType, whitespaceProcessingMethod: WhitespaceProcessing) {
-	let paragraphs: string[] = []
+export class Word {
+	readonly text: string
+	isSentenceFinalizer: boolean
 
-	if (paragraphBreaks === 'single') {
-		paragraphs = text.split(/(\r?\n)+/g)
-	} else if (paragraphBreaks === 'double') {
-		paragraphs = text.split(/(\r?\n)(\r?\n)+/g)
-	} else {
-		throw new Error(`Invalid paragraph break type: '${paragraphBreaks}'`)
+	constructor(text: string, isSentenceFinalizer: boolean) {
+		this.text = text
+		this.isSentenceFinalizer = isSentenceFinalizer
 	}
 
-	paragraphs = paragraphs.map(p => applyWhitespaceProcessing(p.trim(), whitespaceProcessingMethod))
-	paragraphs = paragraphs.filter(p => p.length > 0)
+	get containsOnlyPunctuation() { return !wordCharacterRegExp.test(this.text) && !this.isSymbolWord }
 
-	return paragraphs
-}
+	get isSymbolWord() { return symbolWords.includes(this.text) }
 
-export function applyWhitespaceProcessing(text: string, whitespaceProcessingMethod: WhitespaceProcessing) {
-	if (whitespaceProcessingMethod === 'removeLineBreaks') {
-		return text.replaceAll(/(\r?\n)+/g, ' ')
-	} else if (whitespaceProcessingMethod === 'collapse') {
-		return text.replaceAll(/\s+/g, ' ')
-	} else if (whitespaceProcessingMethod === 'preserve') {
-		return text
-	} else {
-		throw new Error(`Invalid whitespace processing method: '${whitespaceProcessingMethod}'`)
-	}
+	get isPhraseSeperator() { return this.containsOnlyPunctuation && includesAnyOf(this.text, phraseSeparators) }
+
+	get length() { return this.text.length }
 }
 
-export function splitToLines(text: string) {
-	return text.split(/\r?\n/g)
+export type Segment = Sentence | Phrase | Word
+
+export class Fragment {
+	segments: Segment[] = []
+
+	get length() { return sumArray(this.segments, (phrase) => phrase.length) }
+
+	get text() { return this.segments.reduce<string>((result, segment) => result + segment.text, '') }
+
+	get isEmpty() { return this.length == 0 }
+
+	get isNonempty() { return !this.isEmpty }
+
+	get lastSegment() {
+		if (this.isEmpty) {
+			return undefined
+		}
+
+		return this.segments[this.segments.length - 1]
+	}
 }

package/src/nlp/TextNormalizer.ts

@@ -1,5 +1,6 @@
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { substituteCharactersUsingLookup } from '../utilities/StringUtilities.js'
+import { anyOf, buildRegExp, charRange, inputEnd, inputStart, repeated, zeroOrMore } from 'regexp-composer'
 
 export function getNormalizedFragmentsForSpeech(
 	words: string[],
@@ -13,36 +14,6 @@ export function getNormalizedFragmentsForSpeech(
 		return { normalizedFragments: [...nonWhitespaceWords], referenceFragments: [...nonWhitespaceWords] }
 	}
 
-	const numberPattern = /^[0-9][0-9\,\.]*$/
-
-	const fourDigitYearPattern = /^[0-9][0-9][0-9][0-9]$/
-	const fourDigitDecadePattern = /^[0-9][0-9][0-9]0s$/
-
-	const fourDigitYearRangePattern = /^[0-9][0-9][0-9][0-9][\-\–][0-9][0-9][0-9][0-9]$/
-
-	const wordsPrecedingAYear = [
-		'in', 'the', 'a', 'to', 'of', 'since', 'from', 'between', 'by', 'until', 'around', 'before', 'after',
-		'his', 'her', 'year', 'years', 'during', 'copyright', '©', 'early', 'mid', 'late',
-		'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december',
-		'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
-	]
-
-	const wordsPrecedingADecade = [
-		'the', 'in', 'early', 'mid', 'late', 'a'
-	]
-
-	const symbolsPrecedingACurrency = [
-		'$', '€', '£', '¥'
-	]
-
-	const symbolsPrecedingACurrencyAsWords = [
-		'dollars', 'euros', 'pounds', 'yen'
-	]
-
-	const wordsSucceedingACurrency = [
-		'million', 'billion', 'trillion'
-	]
-
 	const normalizedFragments: string[] = []
 	const referenceFragments: string[] = []
 
@@ -56,10 +27,11 @@ export function getNormalizedFragmentsForSpeech(
 		const originalWordIndex = nonWhitespaceWordOriginalIndex[wordIndex]
 		const isFollowedByWhitespace = words[originalWordIndex + 1]?.trim().length === 0
 
-		if ( // Normalize a four digit year pattern, e.g. 'in 1995'.
-			wordsPrecedingAYear.includes(lowerCaseWord) &&
+		if (wordsPrecedingAYear.includes(lowerCaseWord) &&
 			isFollowedByWhitespace &&
-			fourDigitYearPattern.test(nextNonWhitespaceWord)) {
+			fourDigitYearPatternRegExp.test(nextNonWhitespaceWord)) {
+
+ 			// Normalize a four digit year pattern, e.g. 'in 1995'.
 
 			const normalizedString = normalizeFourDigitYearString(nextNonWhitespaceWord)
 
@@ -70,10 +42,12 @@ export function getNormalizedFragmentsForSpeech(
 			referenceFragments.push(nextNonWhitespaceWord)
 
 			wordIndex += 1
-		} else if ( // Normalize a four digit decade pattern, e.g. 'the 1980s'.
+		} else if (
 			wordsPrecedingADecade.includes(lowerCaseWord) &&
 			isFollowedByWhitespace &&
-			fourDigitDecadePattern.test(nextNonWhitespaceWord)) {
+			fourDigitDecadePatternRegExp.test(nextNonWhitespaceWord)) {
+
+ 			// Normalize a four digit decade pattern, e.g. 'the 1980s'.
 
 			const normalizedString = normalizeFourDigitDecadeString(nextNonWhitespaceWord)
 
@@ -84,9 +58,8 @@ export function getNormalizedFragmentsForSpeech(
 			referenceFragments.push(nextNonWhitespaceWord)
 
 			wordIndex += 1
-		} else if ( // Normalize a year range pattern, e.g. '1835-1896', ensure there are no spaces between words
-			fourDigitYearRangePattern.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
-
+		} else if (fourDigitYearRangePatternRegExp.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
+			// Normalize a year range pattern, e.g. '1835-1896', ensure there are no spaces between words
 			normalizedFragments.push(normalizeFourDigitYearString(nonWhitespaceWords[wordIndex]))
 			referenceFragments.push(nonWhitespaceWords[wordIndex])
 
@@ -97,32 +70,37 @@ export function getNormalizedFragmentsForSpeech(
 			referenceFragments.push(nonWhitespaceWords[wordIndex + 2])
 
 			wordIndex += 2
-		} else if ( // Normalize a currency pattern, e.g. '$53.1 million', '€3.53'
-			symbolsPrecedingACurrency.includes(lowerCaseWord) &&
-			!isFollowedByWhitespace &&
-			numberPattern.test(nextNonWhitespaceWord)) {
+		} else if (precedingCurrencyPatternRegExp.test(lowerCaseWord)) {
+			// Normalize a currency pattern with preceding currency symbol, e.g. '€3.53', '$53.1 million',
 
-			let currencyWord = symbolsPrecedingACurrencyAsWords[symbolsPrecedingACurrency.indexOf(lowerCaseWord)]
+			const currencyWord = currencySymbolsAsWords[currencySymbols.indexOf(lowerCaseWord[0])]
 
-			if (wordsSucceedingACurrency.includes(nextNonWhitespaceWords[1]?.toLowerCase())) {
-				const normalizedString = `${nextNonWhitespaceWord} ${nextNonWhitespaceWords[1]} ${currencyWord}`
+			if (wordsFollowingACurrency.includes(nextNonWhitespaceWord?.toLowerCase())) {
+				const normalizedString = `${word.substring(1)} ${nextNonWhitespaceWord} ${currencyWord}`
 
 				normalizedFragments.push(normalizedString)
 
-				const referenceString = `${word}${nextNonWhitespaceWord} ${nextNonWhitespaceWords[1]}`
+				const referenceString = `${word} ${nextNonWhitespaceWord}`
 				referenceFragments.push(referenceString)
 
-				wordIndex += 2
+				wordIndex += 1
 			} else {
-				const normalizedString = `${nextNonWhitespaceWord} ${currencyWord}`
+				const normalizedString = `${word.substring(1)} ${currencyWord}`
 
 				normalizedFragments.push(normalizedString)
 
-				const referenceString = `${word}${nextNonWhitespaceWord}`
+				const referenceString = word
 				referenceFragments.push(referenceString)
-
-				wordIndex += 1
 			}
+		} else if (followingCurrencyPatternRegExp.test(lowerCaseWord)) {
+			const currencyWord = currencySymbolsAsWords[currencySymbols.indexOf(lowerCaseWord[lowerCaseWord.length - 1])]
+
+			const normalizedString = `${word.substring(0, word.length - 1)} ${currencyWord}`
+
+			normalizedFragments.push(normalizedString)
+
+			const referenceString = word
+			referenceFragments.push(referenceString)
 		} else {
 			normalizedFragments.push(word)
 			referenceFragments.push(word)
@@ -156,7 +134,7 @@ export function normalizeFourDigitDecadeString(decadeString: string) {
 	let normalizedString: string
 
 	const isBeforeSecondMillenium = firstTwoDigitsValue < 10
-	const isMilleniumDecade =  firstTwoDigitsValue % 10 == 0 && secondTwoDigitsValue == 0
+	const isMilleniumDecade = firstTwoDigitsValue % 10 == 0 && secondTwoDigitsValue == 0
 
 	if (!isBeforeSecondMillenium && !isMilleniumDecade) {
 		if (secondTwoDigitsValue != 0) {
@@ -229,3 +207,67 @@ export const punctuationSubstitutionLookup: Record<string, string> = {
 	'！': `!`,
 	'¡': `!`,
 }
+
+const wordsPrecedingAYear = [
+	'in', 'the', 'a', 'to', 'of', 'since', 'from', 'between', 'by', 'until', 'around', 'before', 'after',
+	'his', 'her', 'year', 'years', 'during', 'copyright', '©', 'early', 'mid', 'late',
+	'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december',
+	'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec',
+	'winter', 'spring', 'summer', 'fall', 'autumn'
+]
+
+const wordsPrecedingADecade = [
+	'the', 'in', 'early', 'mid', 'late', 'a'
+]
+
+const currencySymbols = [
+	'$', '¥', '€', '£', '₩', '₭', '₽', '₫', '฿', '¢', '₮', '؋', '₦', '₱', '₴', '₪'
+]
+
+const currencySymbolsAsWords = [
+	'dollars', 'yen', 'euros', 'pounds', 'won', 'kip', 'rubles', 'dong', 'baht', 'cents', 'tögrög', 'afghanis', 'naira', 'pesos', 'hryvnia', 'shekels'
+]
+
+const wordsFollowingACurrency = [
+	'million', 'billion', 'trillion'
+]
+
+const arabicNumeralPattern = charRange('0', '9')
+
+const numberPattern = [
+	inputStart,
+	arabicNumeralPattern,
+	zeroOrMore(anyOf(arabicNumeralPattern, ',', '.')),
+	inputEnd
+]
+
+const numberPatternRegExp = buildRegExp(numberPattern)
+
+const precedingCurrencyPattern = [
+	inputStart,
+	anyOf(...currencySymbols),
+	arabicNumeralPattern,
+	zeroOrMore(anyOf(arabicNumeralPattern, ',', '.')),
+	inputEnd
+]
+
+const precedingCurrencyPatternRegExp = buildRegExp(precedingCurrencyPattern)
+
+const followingCurrencyPattern = [
+	inputStart,
+	arabicNumeralPattern,
+	zeroOrMore(anyOf(arabicNumeralPattern, ',', '.')),
+	anyOf(...currencySymbols),
+	inputEnd
+]
+
+const followingCurrencyPatternRegExp = buildRegExp(followingCurrencyPattern)
+
+const fourDigitYearPattern = [inputStart, repeated(4, arabicNumeralPattern), inputEnd]
+const fourDigitYearPatternRegExp = buildRegExp(fourDigitYearPattern)
+
+const fourDigitDecadePattern = [inputStart, repeated(3, arabicNumeralPattern), '0s', inputEnd]
+const fourDigitDecadePatternRegExp = buildRegExp(fourDigitDecadePattern)
+
+const fourDigitYearRangePattern = [inputStart, repeated(4, arabicNumeralPattern), anyOf('-', '–'), repeated(4, arabicNumeralPattern), inputEnd]
+const fourDigitYearRangePatternRegExp = buildRegExp(fourDigitYearRangePattern)

package/src/recognition/AmazonTranscribeSTT.ts

@@ -1,5 +1,5 @@
 import { Item, LanguageCode, StartStreamTranscriptionCommandInput } from '@aws-sdk/client-transcribe-streaming'
-import { wordCharacterPattern } from '../nlp/Segmentation.js'
+import { wordCharacterRegExp } from '../nlp/Segmentation.js'
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
@@ -102,7 +102,7 @@ export async function recgonize(rawAudio: RawAudio, languageCode: string, region
 	for (const event of events) {
 		const text = event.Content!
 
-		if (!wordCharacterPattern.test(text)) {
+		if (!wordCharacterRegExp.test(text)) {
 			continue
 		}
 

package/src/recognition/SileroSTT.ts

@@ -1,5 +1,5 @@
 import { indexOfMax } from '../math/VectorMath.js'
-import { wordCharacterPattern } from '../nlp/Segmentation.js'
+import { wordCharacterRegExp } from '../nlp/Segmentation.js'
 import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { Timeline } from '../utilities/Timeline.js'
@@ -182,7 +182,7 @@ export class SileroSTT {
 		for (let i = 0; i < words.length; i++) {
 			const text = words[i]
 
-			if (!wordCharacterPattern.test(text)) {
+			if (!wordCharacterRegExp.test(text)) {
 				continue
 			}
 

package/src/recognition/WhisperSTT.ts

@@ -18,7 +18,7 @@ import chalk from 'chalk'
 import { XorShift32PRNG } from '../utilities/RandomGenerator.js'
 import { detectSpeechLanguageByParts } from '../api/SpeechLanguageDetection.js'
 import { type Tiktoken } from 'tiktoken/lite'
-import { isPunctuation, isWhitespace, isWord, splitToSentences, splitToWords } from '../nlp/Segmentation.js'
+import { includesPunctuation, isWhitespace, splitToWords } from '../nlp/Segmentation.js'
 import { medianOf5Filter } from '../math/MedianFilter.js'
 import { getDeflateCompressionMetricsForString } from '../utilities/Compression.js'
 import { dmlProviderAvailable, getOnnxSessionOptions, makeOnnxLikeFloat32Tensor, OnnxExecutionProvider, OnnxLikeFloat32Tensor } from '../utilities/OnnxUtilities.js'
@@ -528,25 +528,11 @@ export class Whisper {
 
 		const targetLanguage = task === 'transcribe' ? sourceLanguage : 'en'
 
-		const shouldSplitToSentences = false
 
 		let simplifiedTranscript = ''
 
-		if (shouldSplitToSentences) {
-			const sentences = splitToSentences(transcript, targetLanguage)
-
-			for (const sentence of sentences) {
-				let sentenceWords = await splitToWords(sentence, targetLanguage)
-				sentenceWords = sentenceWords.filter(word => isWord(word))
-
-				simplifiedTranscript += sentenceWords.join(' ')
-				simplifiedTranscript += ' '
-			}
-		} else {
-			let words = await splitToWords(transcript, targetLanguage)
-
-			words = words.map(word => word.trim())
-			words = words.filter(word => isWord(word))
+		{
+			const words = (await splitToWords(transcript, targetLanguage)).nonPunctuationWords
 
 			simplifiedTranscript = words.join(' ')
 		}
@@ -1253,7 +1239,7 @@ export class Whisper {
 				return false
 			}
 
-			return isWhitespace(char) || isPunctuation(char)
+			return isWhitespace(char) || includesPunctuation(char)
 		}
 
 		function startsWithSeparatorCharacter(text: string) {