echogarden 2.6.0 → 2.8.0

package/src/nlp/TextNormalizer.ts

@@ -1,5 +1,6 @@
 import { getShortLanguageCode } from '../utilities/Locale.js'
 import { substituteCharactersUsingLookup } from '../utilities/StringUtilities.js'
+import { anyOf, buildRegExp, charRange, inputEnd, inputStart, repeated, zeroOrMore } from 'regexp-composer'
 
 export function getNormalizedFragmentsForSpeech(
 	words: string[],
@@ -13,36 +14,6 @@ export function getNormalizedFragmentsForSpeech(
 		return { normalizedFragments: [...nonWhitespaceWords], referenceFragments: [...nonWhitespaceWords] }
 	}
 
-	const numberPattern = /^[0-9][0-9\,\.]*$/
-
-	const fourDigitYearPattern = /^[0-9][0-9][0-9][0-9]$/
-	const fourDigitDecadePattern = /^[0-9][0-9][0-9]0s$/
-
-	const fourDigitYearRangePattern = /^[0-9][0-9][0-9][0-9][\-\–][0-9][0-9][0-9][0-9]$/
-
-	const wordsPrecedingAYear = [
-		'in', 'the', 'a', 'to', 'of', 'since', 'from', 'between', 'by', 'until', 'around', 'before', 'after',
-		'his', 'her', 'year', 'years', 'during', 'copyright', '©', 'early', 'mid', 'late',
-		'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december',
-		'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
-	]
-
-	const wordsPrecedingADecade = [
-		'the', 'in', 'early', 'mid', 'late', 'a'
-	]
-
-	const symbolsPrecedingACurrency = [
-		'$', '€', '£', '¥'
-	]
-
-	const symbolsPrecedingACurrencyAsWords = [
-		'dollars', 'euros', 'pounds', 'yen'
-	]
-
-	const wordsSucceedingACurrency = [
-		'million', 'billion', 'trillion'
-	]
-
 	const normalizedFragments: string[] = []
 	const referenceFragments: string[] = []
 
@@ -56,10 +27,11 @@ export function getNormalizedFragmentsForSpeech(
 		const originalWordIndex = nonWhitespaceWordOriginalIndex[wordIndex]
 		const isFollowedByWhitespace = words[originalWordIndex + 1]?.trim().length === 0
 
-		if ( // Normalize a four digit year pattern, e.g. 'in 1995'.
-			wordsPrecedingAYear.includes(lowerCaseWord) &&
+		if (wordsPrecedingAYear.includes(lowerCaseWord) &&
 			isFollowedByWhitespace &&
-			fourDigitYearPattern.test(nextNonWhitespaceWord)) {
+			fourDigitYearPatternRegExp.test(nextNonWhitespaceWord)) {
+
+ 			// Normalize a four digit year pattern, e.g. 'in 1995'.
 
 			const normalizedString = normalizeFourDigitYearString(nextNonWhitespaceWord)
 
@@ -70,10 +42,12 @@ export function getNormalizedFragmentsForSpeech(
 			referenceFragments.push(nextNonWhitespaceWord)
 
 			wordIndex += 1
-		} else if ( // Normalize a four digit decade pattern, e.g. 'the 1980s'.
+		} else if (
 			wordsPrecedingADecade.includes(lowerCaseWord) &&
 			isFollowedByWhitespace &&
-			fourDigitDecadePattern.test(nextNonWhitespaceWord)) {
+			fourDigitDecadePatternRegExp.test(nextNonWhitespaceWord)) {
+
+ 			// Normalize a four digit decade pattern, e.g. 'the 1980s'.
 
 			const normalizedString = normalizeFourDigitDecadeString(nextNonWhitespaceWord)
 
@@ -84,9 +58,8 @@ export function getNormalizedFragmentsForSpeech(
 			referenceFragments.push(nextNonWhitespaceWord)
 
 			wordIndex += 1
-		} else if ( // Normalize a year range pattern, e.g. '1835-1896', ensure there are no spaces between words
-			fourDigitYearRangePattern.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
-
+		} else if (fourDigitYearRangePatternRegExp.test(words.slice(originalWordIndex, originalWordIndex + 3).join(''))) {
+			// Normalize a year range pattern, e.g. '1835-1896', ensure there are no spaces between words
 			normalizedFragments.push(normalizeFourDigitYearString(nonWhitespaceWords[wordIndex]))
 			referenceFragments.push(nonWhitespaceWords[wordIndex])
 
@@ -97,32 +70,37 @@ export function getNormalizedFragmentsForSpeech(
 			referenceFragments.push(nonWhitespaceWords[wordIndex + 2])
 
 			wordIndex += 2
-		} else if ( // Normalize a currency pattern, e.g. '$53.1 million', '€3.53'
-			symbolsPrecedingACurrency.includes(lowerCaseWord) &&
-			!isFollowedByWhitespace &&
-			numberPattern.test(nextNonWhitespaceWord)) {
+		} else if (precedingCurrencyPatternRegExp.test(lowerCaseWord)) {
+			// Normalize a currency pattern with preceding currency symbol, e.g. '€3.53', '$53.1 million',
 
-			let currencyWord = symbolsPrecedingACurrencyAsWords[symbolsPrecedingACurrency.indexOf(lowerCaseWord)]
+			const currencyWord = currencySymbolsAsWords[currencySymbols.indexOf(lowerCaseWord[0])]
 
-			if (wordsSucceedingACurrency.includes(nextNonWhitespaceWords[1]?.toLowerCase())) {
-				const normalizedString = `${nextNonWhitespaceWord} ${nextNonWhitespaceWords[1]} ${currencyWord}`
+			if (wordsFollowingACurrency.includes(nextNonWhitespaceWord?.toLowerCase())) {
+				const normalizedString = `${word.substring(1)} ${nextNonWhitespaceWord} ${currencyWord}`
 
 				normalizedFragments.push(normalizedString)
 
-				const referenceString = `${word}${nextNonWhitespaceWord} ${nextNonWhitespaceWords[1]}`
+				const referenceString = `${word} ${nextNonWhitespaceWord}`
 				referenceFragments.push(referenceString)
 
-				wordIndex += 2
+				wordIndex += 1
 			} else {
-				const normalizedString = `${nextNonWhitespaceWord} ${currencyWord}`
+				const normalizedString = `${word.substring(1)} ${currencyWord}`
 
 				normalizedFragments.push(normalizedString)
 
-				const referenceString = `${word}${nextNonWhitespaceWord}`
+				const referenceString = word
 				referenceFragments.push(referenceString)
-
-				wordIndex += 1
 			}
+		} else if (followingCurrencyPatternRegExp.test(lowerCaseWord)) {
+			const currencyWord = currencySymbolsAsWords[currencySymbols.indexOf(lowerCaseWord[lowerCaseWord.length - 1])]
+
+			const normalizedString = `${word.substring(0, word.length - 1)} ${currencyWord}`
+
+			normalizedFragments.push(normalizedString)
+
+			const referenceString = word
+			referenceFragments.push(referenceString)
 		} else {
 			normalizedFragments.push(word)
 			referenceFragments.push(word)
@@ -156,7 +134,7 @@ export function normalizeFourDigitDecadeString(decadeString: string) {
 	let normalizedString: string
 
 	const isBeforeSecondMillenium = firstTwoDigitsValue < 10
-	const isMilleniumDecade =  firstTwoDigitsValue % 10 == 0 && secondTwoDigitsValue == 0
+	const isMilleniumDecade = firstTwoDigitsValue % 10 == 0 && secondTwoDigitsValue == 0
 
 	if (!isBeforeSecondMillenium && !isMilleniumDecade) {
 		if (secondTwoDigitsValue != 0) {
@@ -229,3 +207,67 @@ export const punctuationSubstitutionLookup: Record<string, string> = {
 	'！': `!`,
 	'¡': `!`,
 }
+
+const wordsPrecedingAYear = [
+	'in', 'the', 'a', 'to', 'of', 'since', 'from', 'between', 'by', 'until', 'around', 'before', 'after',
+	'his', 'her', 'year', 'years', 'during', 'copyright', '©', 'early', 'mid', 'late',
+	'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december',
+	'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec',
+	'winter', 'spring', 'summer', 'fall', 'autumn'
+]
+
+const wordsPrecedingADecade = [
+	'the', 'in', 'early', 'mid', 'late', 'a'
+]
+
+const currencySymbols = [
+	'$', '¥', '€', '£', '₩', '₭', '₽', '₫', '฿', '¢', '₮', '؋', '₦', '₱', '₴', '₪'
+]
+
+const currencySymbolsAsWords = [
+	'dollars', 'yen', 'euros', 'pounds', 'won', 'kip', 'rubles', 'dong', 'baht', 'cents', 'tögrög', 'afghanis', 'naira', 'pesos', 'hryvnia', 'shekels'
+]
+
+const wordsFollowingACurrency = [
+	'million', 'billion', 'trillion'
+]
+
+const arabicNumeralPattern = charRange('0', '9')
+
+const numberPattern = [
+	inputStart,
+	arabicNumeralPattern,
+	zeroOrMore(anyOf(arabicNumeralPattern, ',', '.')),
+	inputEnd
+]
+
+const numberPatternRegExp = buildRegExp(numberPattern)
+
+const precedingCurrencyPattern = [
+	inputStart,
+	anyOf(...currencySymbols),
+	arabicNumeralPattern,
+	zeroOrMore(anyOf(arabicNumeralPattern, ',', '.')),
+	inputEnd
+]
+
+const precedingCurrencyPatternRegExp = buildRegExp(precedingCurrencyPattern)
+
+const followingCurrencyPattern = [
+	inputStart,
+	arabicNumeralPattern,
+	zeroOrMore(anyOf(arabicNumeralPattern, ',', '.')),
+	anyOf(...currencySymbols),
+	inputEnd
+]
+
+const followingCurrencyPatternRegExp = buildRegExp(followingCurrencyPattern)
+
+const fourDigitYearPattern = [inputStart, repeated(4, arabicNumeralPattern), inputEnd]
+const fourDigitYearPatternRegExp = buildRegExp(fourDigitYearPattern)
+
+const fourDigitDecadePattern = [inputStart, repeated(3, arabicNumeralPattern), '0s', inputEnd]
+const fourDigitDecadePatternRegExp = buildRegExp(fourDigitDecadePattern)
+
+const fourDigitYearRangePattern = [inputStart, repeated(4, arabicNumeralPattern), anyOf('-', '–'), repeated(4, arabicNumeralPattern), inputEnd]
+const fourDigitYearRangePatternRegExp = buildRegExp(fourDigitYearRangePattern)

package/src/recognition/AmazonTranscribeSTT.ts

@@ -1,5 +1,5 @@
 import { Item, LanguageCode, StartStreamTranscriptionCommandInput } from '@aws-sdk/client-transcribe-streaming'
-import { wordCharacterPattern } from '../nlp/Segmentation.js'
+import { wordCharacterRegExp } from '../nlp/Segmentation.js'
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
@@ -102,7 +102,7 @@ export async function recgonize(rawAudio: RawAudio, languageCode: string, region
 	for (const event of events) {
 		const text = event.Content!
 
-		if (!wordCharacterPattern.test(text)) {
+		if (!wordCharacterRegExp.test(text)) {
 			continue
 		}
 

package/src/recognition/SileroSTT.ts

@@ -1,5 +1,5 @@
 import { indexOfMax } from '../math/VectorMath.js'
-import { wordCharacterPattern } from '../nlp/Segmentation.js'
+import { wordCharacterRegExp } from '../nlp/Segmentation.js'
 import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { Timeline } from '../utilities/Timeline.js'
@@ -182,7 +182,7 @@ export class SileroSTT {
 		for (let i = 0; i < words.length; i++) {
 			const text = words[i]
 
-			if (!wordCharacterPattern.test(text)) {
+			if (!wordCharacterRegExp.test(text)) {
 				continue
 			}
 

package/src/recognition/WhisperSTT.ts

@@ -18,7 +18,7 @@ import chalk from 'chalk'
 import { XorShift32PRNG } from '../utilities/RandomGenerator.js'
 import { detectSpeechLanguageByParts } from '../api/SpeechLanguageDetection.js'
 import { type Tiktoken } from 'tiktoken/lite'
-import { isPunctuation, isWhitespace, isWord, splitToSentences, splitToWords } from '../nlp/Segmentation.js'
+import { includesPunctuation, isWhitespace, splitToWords } from '../nlp/Segmentation.js'
 import { medianOf5Filter } from '../math/MedianFilter.js'
 import { getDeflateCompressionMetricsForString } from '../utilities/Compression.js'
 import { dmlProviderAvailable, getOnnxSessionOptions, makeOnnxLikeFloat32Tensor, OnnxExecutionProvider, OnnxLikeFloat32Tensor } from '../utilities/OnnxUtilities.js'
@@ -528,25 +528,11 @@ export class Whisper {
 
 		const targetLanguage = task === 'transcribe' ? sourceLanguage : 'en'
 
-		const shouldSplitToSentences = false
 
 		let simplifiedTranscript = ''
 
-		if (shouldSplitToSentences) {
-			const sentences = splitToSentences(transcript, targetLanguage)
-
-			for (const sentence of sentences) {
-				let sentenceWords = await splitToWords(sentence, targetLanguage)
-				sentenceWords = sentenceWords.filter(word => isWord(word))
-
-				simplifiedTranscript += sentenceWords.join(' ')
-				simplifiedTranscript += ' '
-			}
-		} else {
-			let words = await splitToWords(transcript, targetLanguage)
-
-			words = words.map(word => word.trim())
-			words = words.filter(word => isWord(word))
+		{
+			const words = (await splitToWords(transcript, targetLanguage)).nonPunctuationWords
 
 			simplifiedTranscript = words.join(' ')
 		}
@@ -1253,7 +1239,7 @@ export class Whisper {
 				return false
 			}
 
-			return isWhitespace(char) || isPunctuation(char)
+			return isWhitespace(char) || includesPunctuation(char)
 		}
 
 		function startsWithSeparatorCharacter(text: string) {

package/src/subtitles/Subtitles.ts

@@ -6,7 +6,8 @@ import { charactersToWriteAhead } from '../audio/AudioPlayer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { readFileAsUtf8 } from '../utilities/FileSystem.js'
 import { deepClone } from '../utilities/ObjectUtilities.js'
-import { formatHMS, formatMS, startsWithAnyOf } from '../utilities/StringUtilities.js'
+import { formatHMS, formatMS } from '../utilities/StringUtilities.js'
+import { anyOf, buildRegExp, capture, digit, inputStart, matches, notUnicodeProperty, oneOrMore, zeroOrMore } from 'regexp-composer'
 
 export async function subtitlesFileToText(filename: string) {
 	return subtitlesToText(await readFileAsUtf8(filename))
@@ -257,14 +258,12 @@ function getCuesFromTimeline_IsolateSegmentSentence(timeline: Timeline, config:
 			const lineLengthWithNextWord = nextWordExtendedEndOffset - lineStartOffset
 			const wordsRemaining = wordTimeline.length - wordIndex - 1
 
-			const phraseSeparators = [',', '，', '、', ';', ':', '),', '",', '”,', '.', '".', '”.', '."', '.”', '。']
-
 			const lineLengthWithNextWordExceedsMaxLineWidth = lineLengthWithNextWord >= maxLineWidth
 			const lineLengthExceedsHalfMaxLineWidth = lineLength >= maxLineWidth / 2
 
 			const wordsRemainingAreEqualOrLessToMinimumWordsInLine = wordsRemaining <= config.minWordsInLine!
 			const remainingTextExceedsMaxLineWidth = entryText.length - lineStartOffset > maxLineWidth
-			const followingSubstringIsPhraseSeparator = startsWithAnyOf(entryText.substring(wordEndOffset), phraseSeparators)
+			const followingSubstringIsPhraseSeparator = phraseSeparatorRegExp.test(entryText.substring(wordEndOffset))
 
 			const shouldAddNewLine =
 				isLastWord ||
@@ -434,8 +433,7 @@ function getCuesFromTimeline_IsolateLines(timeline: Timeline, config: SubtitlesC
 }
 
 export function tryParseTimeRangePatternWithHours(line: string) {
-	const timeRangePatternWithHours = /^(\d+)\:(\d+)\:(\d+)[\.,](\d+)[ ]*-->[ ]*(\d+)\:(\d+)\:(\d+)[\.,](\d+)/
-	const match = timeRangePatternWithHours.exec(line)
+	const match = timeRangeWithHoursRegExp.exec(line)
 
 	if (!match) {
 		return { startTime: -1, endTime: -1, succeeded: false }
@@ -458,8 +456,7 @@ export function tryParseTimeRangePatternWithHours(line: string) {
 }
 
 export function tryParseTimeRangePatternWithoutHours(line: string) {
-	const timeRangePatternWithHours = /^(\d+)\:(\d+)[\.,](\d+)[ ]*-->[ ]*(\d+)\:(\d+)[\.,](\d+)/
-	const match = timeRangePatternWithHours.exec(line)
+	const match = timeRangeWithoutHoursRegExp.exec(line)
 
 	if (!match) {
 		return { startTime: -1, endTime: -1, succeeded: false }
@@ -514,6 +511,70 @@ function cueObjectToText(cue: Cue, cueIndex: number, config: SubtitlesConfig) {
 	return outText
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Patterns
+////////////////////////////////////////////////////////////////////////////////////////////////////////
+const timeRangeWithHoursRegExp = buildRegExp([
+	inputStart,
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+
+	anyOf('.', ','),
+	capture(oneOrMore(digit)),
+
+	zeroOrMore(' '),
+	'-->',
+	zeroOrMore(' '),
+
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+
+	anyOf('.', ','),
+	capture(oneOrMore(digit)),
+])
+
+const timeRangeWithoutHoursRegExp = buildRegExp([
+	inputStart,
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+
+	anyOf('.', ','),
+	capture(oneOrMore(digit)),
+
+	zeroOrMore(' '),
+	'-->',
+	zeroOrMore(' '),
+
+	capture(oneOrMore(digit)),
+	':',
+	capture(oneOrMore(digit)),
+
+	anyOf('.', ','),
+	capture(oneOrMore(digit)),
+])
+
+const phraseSeparatorCharacters = [',', '，', '、', ';', ':', '),', '",', '”,']
+
+const phraseSeparatorRegExp = buildRegExp(
+	matches([
+		inputStart,
+		anyOf(...phraseSeparatorCharacters)
+	], {
+		ifPrecededBy: notUnicodeProperty('Decimal_Number'),
+		ifFollowedBy: notUnicodeProperty('Decimal_Number'),
+	})
+)
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Types
+////////////////////////////////////////////////////////////////////////////////////////////////////////
 export type Cue = {
 	lines: string[]
 	startTime: number

package/src/synthesis/ElevenLabsTTS.ts

@@ -5,7 +5,7 @@ import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { decodeBase64 } from '../encodings/Base64.js'
-import { isWordOrSymbolWord, splitToWords } from '../nlp/Segmentation.js'
+import { splitToWords } from '../nlp/Segmentation.js'
 import { Timeline } from '../utilities/Timeline.js'
 
 const log = logToStderr
@@ -79,24 +79,25 @@ export async function synthesize(text: string, voiceId: string, language: string
 		logger.start('Create timeline from returned character timings')
 
 		const referenceText = characters.join('')
-		const words = (await splitToWords(referenceText, language)).filter(w => isWordOrSymbolWord(w))
+		const wordSequence = await splitToWords(referenceText, language)
 
 		timeline = []
 
-		let offset = 0
+		for (const wordEntry of wordSequence.entries) {
+			const wordText = wordEntry.text
 
-		for (const word of words) {
-			const wordStartIndex = referenceText.indexOf(word, offset)
-			const wordEndIndex = wordStartIndex + word.length
+			const wordStartOffset = wordEntry.startOffset
+			const wordEndOffset = wordEntry.endOffset
+
+			const startTime = characterStartTimes[wordStartOffset]
+			const endTime = characterEndTimes[wordEndOffset] ?? characterEndTimes[wordEndOffset - 1]
 
 			timeline.push({
 				type: 'word',
-				text: word,
-				startTime: characterStartTimes[wordStartIndex],
-				endTime: characterEndTimes[wordEndIndex] ?? characterEndTimes[wordEndIndex - 1]
+				text: wordText,
+				startTime,
+				endTime,
 			})
-
-			offset = wordEndIndex
 		}
 	}
 

package/src/synthesis/EspeakTTS.ts

@@ -1,17 +1,18 @@
 import { concatFloat32Arrays, logToStderr, formatObjectToString } from '../utilities/Utilities.js'
 import { int16PcmToFloat32 } from '../audio/AudioBufferConversion.js'
 import { Logger } from '../utilities/Logger.js'
-import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js'
 import { RawAudio, getEmptyRawAudio } from '../audio/AudioUtilities.js'
 import { getNormalizedFragmentsForSpeech, simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
 import { ipaPhoneToKirshenbaum } from '../nlp/PhoneConversion.js'
-import { splitToWords, wordCharacterPattern } from '../nlp/Segmentation.js'
+import { splitToWords, wordCharacterRegExp } from '../nlp/Segmentation.js'
 import { Lexicon, tryGetFirstLexiconSubstitution } from '../nlp/Lexicon.js'
 import { phonemizeSentence } from '../nlp/EspeakPhonemizer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { escapeHtml } from '../encodings/HtmlEscape.js'
 
+import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
+
 const log = logToStderr
 
 let espeakInstance: any
@@ -37,7 +38,7 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 	fragments = []
 	preprocessedFragments = []
 
-	let words = await splitToWords(text, language)
+	let words = (await splitToWords(text, language)).wordArray
 
 	// Merge repeating non-words to a single word to work around eSpeak bug
 	{
@@ -51,7 +52,7 @@ export async function preprocessAndSynthesize(text: string, language: string, es
 				i > 0 &&
 				currentWord === previousWord &&
 				!['[', ']'].includes(currentWord) && // Work around eSpeak-NG marker bug with repeating squared brackets
-				!wordCharacterPattern.test(currentWord)) {
+				!wordCharacterRegExp.test(currentWord)) {
 
 				wordsWithMerges[wordsWithMerges.length - 1] += currentWord
 			} else {
@@ -515,12 +516,12 @@ export async function textToPhonemes(text: string, voice: string, useIPA = true)
 	const { instance, module } = await getEspeakInstance()
 	const textPtr = instance.convert_to_phonemes(text, useIPA)
 
-	const wasmMemory = new WasmMemoryManager(module)
+	const wasmHeap = wrapEmscriptenModuleHeap(module)
 
-	const resultRef = wasmMemory.wrapNullTerminatedUtf8String(textPtr.ptr)
-	const result = resultRef.getValue()
+	const resultRef = wasmHeap.wrapNullTerminatedUtf8String(textPtr.ptr)
+	const result = resultRef.value
 
-	wasmMemory.freeAll()
+	resultRef.free()
 
 	return result
 }

package/src/synthesis/KokoroTTS.ts

@@ -108,9 +108,6 @@ export class KokoroTTS {
 		const {
 			referenceSynthesizedAudio,
 			referenceTimeline,
-			fragments,
-			phonemizedFragmentsSubstitutions,
-			phonemizedSentence
 		} = await Espeak.preprocessAndSynthesize(sentenceText, voiceLanguage, espeakOptions, lexicons)
 
 		logger.end()
@@ -132,13 +129,28 @@ export class KokoroTTS {
 
 		const allTokenIds: number[] = []
 
-		for (let phraseIndex = 0; phraseIndex < phonemizedSentence.length; phraseIndex++) {
-			const phrase = phonemizedSentence[phraseIndex]
+		for (let clauseIndex = 0; clauseIndex < referenceTimeline.length; clauseIndex++) {
+			const clauseEntry = referenceTimeline[clauseIndex]
 
-			for (let wordIndex = 0; wordIndex < phrase.length; wordIndex++) {
-				const word = phrase[wordIndex]
+			if (!clauseEntry.timeline) {
+				continue
+			}
+
+			const wordTimelineForClause = clauseEntry.timeline
+
+			for (let wordIndex = 0; wordIndex < wordTimelineForClause.length; wordIndex++) {
+				const wordEntry = wordTimelineForClause[wordIndex]
+				const wordText = wordEntry.text
+
+				if (!wordEntry.timeline) {
+					continue
+				}
+
+				const tokenTimelineForWord = wordEntry.timeline
+				const phonemeTimelineForWord = tokenTimelineForWord.flatMap(tokenEntry => tokenEntry.timeline ?? [])
 
-				for (const phoneme of word) {
+				for (const phonemeEntry of phonemeTimelineForWord) {
+					const phoneme = phonemeEntry.text
 					let processedPhoneme = phoneme
 
 					if (voicePrimaryLanguageShort === 'en') {
@@ -179,7 +191,7 @@ export class KokoroTTS {
 
 						if (false) {
 							// Workaround a word having only 'I' not being pronounced at some cases
-							if (processedPhoneme === 'I' && word.length === 1) {
+							if (processedPhoneme === 'I' && wordText.length === 1) {
 								processedPhoneme = 'aɪ'
 								//processedPhoneme = 'ˌI'
 							}
@@ -196,10 +208,10 @@ export class KokoroTTS {
 					}
 				}
 
-				if (wordIndex < phrase.length - 1) {
+				if (wordIndex < wordTimelineForClause.length - 1) {
 					allTokenIds.push(wordBreakTokenId)
 				} else {
-					if (phraseIndex < phonemizedSentence.length - 1) {
+					if (clauseIndex < referenceTimeline.length - 1) {
 						allTokenIds.push(phraseBreakTokenId)
 						allTokenIds.push(wordBreakTokenId)
 					}
@@ -209,7 +221,7 @@ export class KokoroTTS {
 
 		allTokenIds.push(sentenceEndTokenId)
 
-		const maxPartLength = 509
+		const maxPartLength = 500
 
 		const parts: number[][] = []
 

package/src/synthesis/SapiTTS.ts

@@ -1,7 +1,6 @@
 import { SynthesisVoice } from '../api/API.js'
-import { decodeToChannels } from '../audio/AudioBufferConversion.js'
+import { decodeToChannels, SampleFormat } from '../audio/AudioBufferConversion.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
-import { SampleFormat } from '../codecs/WaveCodec.js'
 import { getShortLanguageCode, lcidToIsoLanguageCode } from '../utilities/Locale.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'