echogarden 1.0.0 → 1.0.2

package/src/api/Alignment.ts

@@ -11,7 +11,8 @@ import { WhisperOptions } from '../recognition/WhisperSTT.js'
 import chalk from 'chalk'
 import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
 import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
-
+import { EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
+import { isWord } from '../nlp/Segmentation.js'
 
 const log = logToStderr
 
@@ -145,7 +146,12 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			logger.end()
 
 			// Recognize source audio
-			const { transcript: recognizedTranscript, wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions)
+			let { wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions)
+
+			logger.log('')
+
+			// Remove non-word entries from recognition timeline
+			recognitionTimeline = recognitionTimeline.filter(entry => isWord(entry.text))
 
 			// Synthesize the ground-truth transcript and get its timeline
 			logger.start('Synthesize ground-truth transcript with eSpeak')
@@ -153,15 +159,8 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			const {
 				referenceRawAudio,
 				referenceTimeline,
-			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, true)
-
-			// Synthesize the recognized transcript and get its timeline
-			logger.start('Synthesize recognized transcript with eSpeak')
-
-			const {
-				referenceRawAudio: synthesizedRecognizedTranscriptRawAudio,
-				referenceTimeline: synthesizedRecognitionTimeline
-			} = await createAlignmentReferenceUsingEspeak(recognizedTranscript, language, undefined, undefined, true)
+				espeakVoice,
+			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, false)
 
 			logger.end()
 
@@ -169,6 +168,13 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 			const phoneAlignmentMethod = options.dtw!.phoneAlignmentMethod!
 
+			const espeakOptions: EspeakOptions = {
+				...defaultEspeakOptions,
+				voice: espeakVoice,
+				useKlatt: false,
+				insertSeparators: true
+			}
+
 			// Align the ground-truth transcript and the recognized transcript
 			mappedTimeline = await alignUsingDtwWithRecognitionReference(
 				sourceRawAudio,
@@ -176,11 +182,10 @@ export async function align(input: AudioSourceParam, transcript: string, options
 				referenceTimeline,
 
 				recognitionTimeline,
-				synthesizedRecognizedTranscriptRawAudio,
-				synthesizedRecognitionTimeline,
 
 				granularities,
 				windowDurations,
+				espeakOptions,
 				phoneAlignmentMethod)
 
 			break
@@ -346,10 +351,10 @@ export const defaultAlignmentOptions: AlignmentOptions = {
 			topCandidateCount: 5,
 			punctuationThreshold: 0.2,
 			maxTokensPerPart: 250,
-			autoPromptParts: true,
+			autoPromptParts: false,
 			suppressRepetition: true,
+			decodeTimestampTokens: true,
 			seed: undefined,
-			decodeTimestampTokens: false,
 		}
 	},
 

package/src/dsp/FFT.ts

@@ -122,7 +122,7 @@ export async function stiftr(binsForFrames: Float32Array[], fftOrder: number, wi
 
 	wasmMemory.freeAll()
 
-	// Divide by sum of weight squares for each samples
+	// Divide each output sample by the sum of squared weights
 	for (let i = 0; i < outSamples.length; i++) {
 		outSamples[i] /= sumOfSquaredWeightsForSample[i] + 1e-8
 	}

package/src/nlp/Segmentation.ts

@@ -8,8 +8,9 @@ import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
 
 const log = logToStderr
 
-export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u
-export const punctuationPattern = /[\p{Punctuation}]/u
+export const wordCharacterPattern = /[\p{Letter}\p{Number}]+/u
+export const punctuationPattern = /[\p{Punctuation}]+/u
+
 export const phraseSeparators = [',', ';', ':']
 export const sentenceSeparators = ['.', '?', '!']
 export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '=', '¼', '½', '¾']
@@ -18,13 +19,20 @@ export function isWordOrSymbolWord(str: string) {
 	return isWord(str) || symbolWords.includes(str)
 }
 
+export function isSymbolWord(str: string) {
+	return symbolWords.includes(str.trim())
+}
+
 export function isWord(str: string) {
-	str = str.trim()
-	return wordCharacterPattern.test(str) || symbolWords.includes(str)
+	return wordCharacterPattern.test(str.trim())
 }
 
 export function isPunctuation(str: string) {
-	return punctuationPattern.test(str)
+	return punctuationPattern.test(str.trim())
+}
+
+export function isWhitespace(str: string) {
+	return str.trim().length === 0
 }
 
 export class Sentence {

package/src/recognition/WhisperCppSTT.ts

@@ -288,13 +288,14 @@ async function parseResultObject(resultObject: WhisperCppVerboseResult, modelNam
 
 	const allTokenIds = tokenTimeline.map(entry => entry.id!)
 	const transcript = whisper.tokensToText(allTokenIds).trim()
+	const language = resultObject.result.language
 
-	let timeline = whisper.tokenTimelineToWordTimeline(tokenTimeline)
+	const timeline = whisper.tokenTimelineToWordTimeline(tokenTimeline, language)
 
 	return {
 		transcript,
 		timeline,
-		language: resultObject.result.language
+		language
 	}
 }
 

package/src/recognition/WhisperSTT.ts

@@ -19,6 +19,7 @@ import chalk from 'chalk'
 import { XorShift32RNG } from '../utilities/RandomGenerator.js'
 import { detectSpeechLanguageByParts } from '../api/LanguageDetection.js'
 import { type Tiktoken } from 'tiktoken/lite'
+import { isPunctuation, isWhitespace } from '../nlp/Segmentation.js'
 
 export async function recognize(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, task: WhisperTask, sourceLanguage: string, options: WhisperOptions) {
 	if (sourceRawAudio.sampleRate != 16000) {
@@ -398,7 +399,7 @@ export class Whisper {
 			logger.end()
 		}
 
-		timeline = this.tokenTimelineToWordTimeline(timeline)
+		timeline = this.tokenTimelineToWordTimeline(timeline, language)
 
 		const transcript = this.tokensToText(allDecodedTokens).trim()
 
@@ -438,7 +439,7 @@ export class Whisper {
 		const alignmentPath = await this.findAlignmentPathFromQKs(crossAttentionQKs, tokens, 0, audioFrameCount)//, this.getAlignmentHeadIndexes())
 		let timeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, tokens, 0, audioDuration)
 
-		timeline = this.tokenTimelineToWordTimeline(timeline)
+		timeline = this.tokenTimelineToWordTimeline(timeline, language)
 
 		logger.end()
 
@@ -1022,33 +1023,40 @@ export class Whisper {
 		}
 	}
 
-	tokenTimelineToWordTimeline(tokenTimeline: Timeline) {
-		const separatorChars =
-			[' ', '–', '一', '，', '、', '|', '/', '\\', ';', '"', '“', '”', '…', '(', ')', '[', ']', '{', '}']
+	tokenTimelineToWordTimeline(tokenTimeline: Timeline, language: string): Timeline {
+		function isSeparatorCharacter(char: string) {
+			const nonSeparatingPunctuation = [`'`, `-`, `.`, `·`, `•`]
 
-		function startsWithSeparatingPunctuation(text: string) {
-			return separatorChars.some(char => text.startsWith(char))
+			if (nonSeparatingPunctuation.includes(char)) {
+				return false
+			}
+
+			return isWhitespace(char) || isPunctuation(char)
+		}
+
+		function startsWithSeparatorCharacter(text: string) {
+			return isSeparatorCharacter(text[0])
 		}
 
-		function isSeparatorPunctuation(text: string) {
-			return separatorChars.includes(text)
+		function endsWithSeparatorCharacter(text: string) {
+			return isSeparatorCharacter(text[text.length - 1])
 		}
 
 		const resultTimeline: Timeline = []
 
-		const groups: TimelineEntry[][] = []
+		let groups: TimelineEntry[][] = []
 
-		for (let i = 0; i < tokenTimeline.length; i++) {
-			const entry = tokenTimeline[i]
-			const previousEntry = i > 0 ? tokenTimeline[i - 1] : undefined
+		for (let tokenIndex = 0; tokenIndex < tokenTimeline.length; tokenIndex++) {
+			const entry = tokenTimeline[tokenIndex]
+			const previousEntry = tokenIndex > 0 ? tokenTimeline[tokenIndex - 1] : undefined
 
 			const text = entry.text
 			const previousEntryText = previousEntry?.text
 
 			if (groups.length == 0 ||
 				text === '' ||
-				startsWithSeparatingPunctuation(text) ||
-				(previousEntryText != null && isSeparatorPunctuation(previousEntryText))) {
+				startsWithSeparatorCharacter(text) ||
+				(previousEntryText != null && endsWithSeparatorCharacter(previousEntryText))) {
 
 				groups.push([entry])
 			} else {
@@ -1056,8 +1064,28 @@ export class Whisper {
 			}
 		}
 
+		const newGroups: TimelineEntry[][] = []
+
+		for (let groupIndex = 0; groupIndex < groups.length - 1; groupIndex++) {
+			const group = groups[groupIndex]
+			const nextGroup = groups[groupIndex + 1]
+
+			if (
+				group.length > 1 &&
+				group[group.length - 1].text === '.' &&
+				[' ', '['].includes(nextGroup[0].text[0])) {
+
+				newGroups.push(group.slice(0, group.length - 1))
+				newGroups.push(group.slice(group.length - 1))
+			} else {
+				newGroups.push(group)
+			}
+		}
+
+		groups = newGroups
+
 		for (const group of groups) {
-			const groupText = this.tokensToText(group.map(entry => entry.id!))
+			let groupText = this.tokensToText(group.map(entry => entry.id!))
 
 			if (groupText === '') {
 				continue
@@ -1755,8 +1783,8 @@ export interface WhisperOptions {
 	autoPromptParts?: boolean
 	maxTokensPerPart?: number
 	suppressRepetition?: boolean
-	seed?: number
 	decodeTimestampTokens?: boolean
+	seed?: number
 }
 
 export const defaultWhisperOptions: WhisperOptions = {
@@ -1768,6 +1796,6 @@ export const defaultWhisperOptions: WhisperOptions = {
 	autoPromptParts: true,
 	maxTokensPerPart: 250,
 	suppressRepetition: true,
+	decodeTimestampTokens: true,
 	seed: undefined,
-	decodeTimestampTokens: false,
 }

package/src/subtitles/Subtitles.ts

@@ -200,7 +200,7 @@ function getCuesFromTimeline_IsolateSegmentSentence(timeline: Timeline, config:
 			continue
 		}
 
-		const wordTimeline = entry.timeline!.filter(entry => isWord(entry.text))
+		const wordTimeline = entry.timeline!.filter(entry => isWordOrSymbolWord(entry.text))
 
 		// First, add word start and end offsets for all word entries
 		let lastWordEndOffset = 0

package/src/tests/Test.ts

@@ -1,6 +1,7 @@
-import { getRepetitionScoreRelativeToFirstSubstring, logToStderr, setupProgramTerminationListeners } from '../utilities/Utilities.js'
+import { getRepetitionScoreRelativeToFirstSubstring, logToStderr, setupProgramTerminationListeners, writeToStderr } from '../utilities/Utilities.js'
 import { makeTarballsForInstalledPackages } from '../utilities/TarballMaker.js'
 import { testEspeakSynthesisWithPrePhonemizedInputs, testKirshenbaumPhonemization } from '../synthesis/EspeakTTS.js'
+import { isPunctuation } from '../nlp/Segmentation.js'
 
 const log = logToStderr
 
@@ -21,4 +22,19 @@ setupProgramTerminationListeners()
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'c', 'a', 'b', 'a', 'c', 'a'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'b', 'a', 'a', 'a', 'b'])
 
+/*
+const allPunctuationChars: string[] = []
+
+for (let i = 0; i < 65536; i++) {
+	const char = String.fromCodePoint(i)
+
+	if (isPunctuation(char)) {
+		allPunctuationChars.push(char)
+
+		writeToStderr(`${char} `)
+	}
+}
+*/
+
 process.exit(0)
+