echogarden 1.0.1 → 1.0.3

package/src/api/Synthesis.ts

@@ -22,8 +22,8 @@ import { shouldCancelCurrentTask } from '../server/Worker.js'
 import chalk from 'chalk'
 import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
 import { type EspeakOptions } from '../synthesis/EspeakTTS.js'
-import { OpenAICloudTTSOptions, defaultOpenAICloudTTSOptions } from '../synthesis/OpenAICloudTTS.js'
-import { ElevenlabsTTSOptions, defaultElevenlabsTTSOptions } from '../synthesis/ElevenlabsTTS.js'
+import { type OpenAICloudTTSOptions, defaultOpenAICloudTTSOptions } from '../synthesis/OpenAICloudTTS.js'
+import { type ElevenlabsTTSOptions, defaultElevenlabsTTSOptions } from '../synthesis/ElevenlabsTTS.js'
 
 const log = logToStderr
 
@@ -1036,7 +1036,7 @@ export interface SynthesisOptions {
 		rate?: number
 		pitch?: number
 		pitchRange?: number
-		
+
 		useKlatt?: boolean
 		insertSeparators?: boolean
 	}

package/src/api/Translation.ts

@@ -8,13 +8,13 @@ import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTi
 import { defaultWhisperOptions, type WhisperOptions } from '../recognition/WhisperSTT.js'
 import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
 import { EngineMetadata } from './Common.js'
-import { SpeechLanguageDetectionOptions, detectSpeechLanguage } from './API.js'
+import { type SpeechLanguageDetectionOptions, detectSpeechLanguage } from './API.js'
 import chalk from 'chalk'
 import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
 
 import * as API from './API.js'
-import { OpenAICloudSTTOptions, defaultOpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
-import { WhisperCppOptions, defaultWhisperCppOptions } from '../recognition/WhisperCppSTT.js'
+import { type OpenAICloudSTTOptions, defaultOpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
+import { type WhisperCppOptions, defaultWhisperCppOptions } from '../recognition/WhisperCppSTT.js'
 
 const log = logToStderr
 

package/src/dsp/FFT.ts

@@ -122,7 +122,7 @@ export async function stiftr(binsForFrames: Float32Array[], fftOrder: number, wi
 
 	wasmMemory.freeAll()
 
-	// Divide by sum of weight squares for each samples
+	// Divide each output sample by the sum of squared weights
 	for (let i = 0; i < outSamples.length; i++) {
 		outSamples[i] /= sumOfSquaredWeightsForSample[i] + 1e-8
 	}

package/src/nlp/Segmentation.ts

@@ -10,6 +10,7 @@ const log = logToStderr
 
 export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u
 export const punctuationPattern = /[\p{Punctuation}]/u
+
 export const phraseSeparators = [',', ';', ':']
 export const sentenceSeparators = ['.', '?', '!']
 export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '=', '¼', '½', '¾']
@@ -18,13 +19,20 @@ export function isWordOrSymbolWord(str: string) {
 	return isWord(str) || symbolWords.includes(str)
 }
 
+export function isSymbolWord(str: string) {
+	return symbolWords.includes(str.trim())
+}
+
 export function isWord(str: string) {
-	str = str.trim()
-	return wordCharacterPattern.test(str) || symbolWords.includes(str)
+	return wordCharacterPattern.test(str.trim())
 }
 
 export function isPunctuation(str: string) {
-	return punctuationPattern.test(str)
+	return punctuationPattern.test(str.trim())
+}
+
+export function isWhitespace(str: string) {
+	return str.trim().length === 0
 }
 
 export class Sentence {

package/src/recognition/SileroSTT.ts

@@ -1,11 +1,11 @@
 import { indexOfMax } from '../math/VectorMath.js'
 import { wordCharacterPattern } from '../nlp/Segmentation.js'
-import Onnx from 'onnxruntime-node'
 import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { RawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
 import { readAndParseJsonFile, readFile } from '../utilities/FileSystem.js'
+import type * as Onnx from 'onnxruntime-node'
 import path from 'path'
 
 const log = logToStderr
@@ -14,6 +14,8 @@ export async function recognize(rawAudio: RawAudio, modelDirectory: string) {
 	const logger = new Logger()
 	logger.start('Create ONNX inference session')
 
+	const Onnx = await import('onnxruntime-node')
+
 	const modelPath = path.join(modelDirectory, 'model.onnx')
 	const labelsPath = path.join(modelDirectory, 'labels.json')
 

package/src/recognition/WhisperCppSTT.ts

@@ -288,13 +288,14 @@ async function parseResultObject(resultObject: WhisperCppVerboseResult, modelNam
 
 	const allTokenIds = tokenTimeline.map(entry => entry.id!)
 	const transcript = whisper.tokensToText(allTokenIds).trim()
+	const language = resultObject.result.language
 
-	let timeline = whisper.tokenTimelineToWordTimeline(tokenTimeline)
+	const timeline = whisper.tokenTimelineToWordTimeline(tokenTimeline, language)
 
 	return {
 		transcript,
 		timeline,
-		language: resultObject.result.language
+		language
 	}
 }
 

package/src/recognition/WhisperSTT.ts

@@ -1,8 +1,8 @@
-import Onnx from 'onnxruntime-node'
+import type * as Onnx from 'onnxruntime-node'
 
 import { Logger } from '../utilities/Logger.js'
 import { computeMelSpectogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectogram.js'
-import { clip, getIntegerRange, getRepetitionScoreRelativeToFirstSubstring, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
+import { clip, getIntegerRange, getRepetitionScoreRelativeToFirstSubstring, getUTF32Chars, logToStderr, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
 import { indexOfMax, logOfVector, logSumExp, meanOfVector, medianFilter, softmax, stdDeviationOfVector } from '../math/VectorMath.js'
 
 import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
@@ -19,6 +19,7 @@ import chalk from 'chalk'
 import { XorShift32RNG } from '../utilities/RandomGenerator.js'
 import { detectSpeechLanguageByParts } from '../api/LanguageDetection.js'
 import { type Tiktoken } from 'tiktoken/lite'
+import { isPunctuation, isWhitespace } from '../nlp/Segmentation.js'
 
 export async function recognize(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, task: WhisperTask, sourceLanguage: string, options: WhisperOptions) {
 	if (sourceRawAudio.sampleRate != 16000) {
@@ -134,8 +135,6 @@ export class Whisper {
 
 		timestampTokensStart: number
 		timestampTokensEnd: number
-
-		suppressedTokens: Set<number>
 	}
 
 	randomGen: XorShift32RNG
@@ -162,16 +161,7 @@ export class Whisper {
 
 				timestampTokensStart: 50364,
 				timestampTokensEnd: 50364 + 1501,
-
-				suppressedTokens: new Set()
 			}
-
-			const suppressedTextTokens = [1, 2, 6, 7, 8, 9, 10, 12, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254]
-			const suppressedMetadataTokens = [50256, ...getIntegerRange(50258, 50364)]
-
-			const suppressedTokens = [...suppressedTextTokens, ...suppressedMetadataTokens]
-
-			this.tokenConfig.suppressedTokens = new Set(suppressedTokens)
 		} else {
 			this.tokenConfig = {
 				endOfTextToken: 50256,
@@ -188,16 +178,7 @@ export class Whisper {
 
 				timestampTokensStart: 50363,
 				timestampTokensEnd: 50363 + 1501,
-
-				suppressedTokens: new Set()
 			}
-
-			const suppressedTextTokens = [1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, 705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, 1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211, 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786, 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146]
-			const suppressedMetadataTokens = [...getIntegerRange(50257, 50363)]
-
-			const suppressedTokens = [...suppressedTextTokens, ...suppressedMetadataTokens]
-
-			this.tokenConfig.suppressedTokens = new Set(suppressedTokens)
 		}
 
 		this.randomGen = new XorShift32RNG(rngSeed)
@@ -282,6 +263,8 @@ export class Whisper {
 
 		const encoderFilePath = path.join(this.modelDir, 'encoder.onnx')
 
+		const Onnx = await import('onnxruntime-node')
+
 		this.audioEncoder = await Onnx.InferenceSession.create(encoderFilePath, this.onnxOptions)
 
 		logger.end()
@@ -298,6 +281,8 @@ export class Whisper {
 
 		const decoderFilePath = path.join(this.modelDir, 'decoder.onnx')
 
+		const Onnx = await import('onnxruntime-node')
+
 		this.textDecoder = await Onnx.InferenceSession.create(decoderFilePath, this.onnxOptions)
 
 		logger.end()
@@ -398,7 +383,7 @@ export class Whisper {
 			logger.end()
 		}
 
-		timeline = this.tokenTimelineToWordTimeline(timeline)
+		timeline = this.tokenTimelineToWordTimeline(timeline, language)
 
 		const transcript = this.tokensToText(allDecodedTokens).trim()
 
@@ -438,7 +423,7 @@ export class Whisper {
 		const alignmentPath = await this.findAlignmentPathFromQKs(crossAttentionQKs, tokens, 0, audioFrameCount)//, this.getAlignmentHeadIndexes())
 		let timeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, tokens, 0, audioDuration)
 
-		timeline = this.tokenTimelineToWordTimeline(timeline)
+		timeline = this.tokenTimelineToWordTimeline(timeline, language)
 
 		logger.end()
 
@@ -462,6 +447,8 @@ export class Whisper {
 		const initialTokens = [sotToken]
 		const offset = 0
 
+		const Onnx = await import('onnxruntime-node')
+
 		const initialKvDimensions = this.getKvDimensions(1, initialTokens.length)
 		const kvCacheTensor = new Onnx.Tensor('float32', new Float32Array(initialKvDimensions[0] * initialKvDimensions[1] * initialKvDimensions[2] * initialKvDimensions[3]), initialKvDimensions)
 
@@ -514,21 +501,23 @@ export class Whisper {
 
 		const logger = new Logger()
 
+		const allowedPunctuationMarks = this.getAllowedPunctuationMarks()
+
 		await logger.startAsync('Decode text tokens with Whisper decoder model')
 
 		options = extendDeep(defaultWhisperOptions, options)
 
+		const Onnx = await import('onnxruntime-node')
+
 		const endOfTextToken = this.tokenConfig.endOfTextToken
 
 		const timestampTokensStart = this.tokenConfig.timestampTokensStart
-		const suppressedTokens = this.tokenConfig.suppressedTokens
+		const suppressedTokens = new Set(this.getSuppressedTokens())
 
 		const spaceToken = this.textToTokens(' ')[0]
 
 		const maxDecodedTokenCount = options.maxTokensPerPart!
 
-		//const suppressedTokensText = Array.from(this.tokenConfig.suppressedTokens).map(token => this.tokenToText(token, true))
-
 		let decodedTokens = initialTokens.slice()
 		const initialKvDimensions = this.getKvDimensions(1, decodedTokens.length)
 		let kvCacheTensor = new Onnx.Tensor('float32', new Float32Array(initialKvDimensions[0] * initialKvDimensions[1] * initialKvDimensions[2] * initialKvDimensions[3]), initialKvDimensions)
@@ -688,7 +677,7 @@ export class Whisper {
 						return maxScore
 					})
 
-					const thresholdRepetitionScore = 3
+					const thresholdRepetitionScore = 4
 
 					if (topCandidatesRepetitionScores.every(score => score >= thresholdRepetitionScore)) {
 						const indexOfMaxScore = topCandidatesRepetitionScores.indexOf(Math.max(...topCandidatesRepetitionScores))
@@ -716,7 +705,7 @@ export class Whisper {
 				const rankOfPromisingPunctuationToken = topCandidates.findIndex((entry, index) => {
 					const tokenText = this.tokenToText(entry.token).trim()
 
-					const isPunctuationToken = [',', '，', '、', '.', '。', '!', '?'].includes(tokenText)
+					const isPunctuationToken = allowedPunctuationMarks.includes(tokenText)
 
 					if (!isPunctuationToken) {
 						return false
@@ -793,6 +782,8 @@ export class Whisper {
 	async inferCrossAttentionQKs(tokens: number[], audioFeatures: Onnx.Tensor) {
 		const offset = 0
 
+		const Onnx = await import('onnxruntime-node')
+
 		const tokensTensor = new Onnx.Tensor('int64', new BigInt64Array(tokens.map(token => BigInt(token))), [1, tokens.length])
 		const offsetTensor = new Onnx.Tensor('int64', new BigInt64Array([BigInt(offset)]), [])
 
@@ -841,6 +832,8 @@ export class Whisper {
 	async encodeAudio(rawAudio: RawAudio) {
 		await this.initializeEncoderSessionIfNeeded()
 
+		const Onnx = await import('onnxruntime-node')
+
 		const logger = new Logger()
 
 		const audioSamples = rawAudio.audioChannels[0]
@@ -1022,33 +1015,40 @@ export class Whisper {
 		}
 	}
 
-	tokenTimelineToWordTimeline(tokenTimeline: Timeline) {
-		const separatorChars =
-			[' ', '–', '一', '，', '、', '|', '/', '\\', ';', '"', '“', '”', '…', '(', ')', '[', ']', '{', '}']
+	tokenTimelineToWordTimeline(tokenTimeline: Timeline, language: string): Timeline {
+		function isSeparatorCharacter(char: string) {
+			const nonSeparatingPunctuation = [`'`, `-`, `.`, `·`, `•`]
+
+			if (nonSeparatingPunctuation.includes(char)) {
+				return false
+			}
 
-		function startsWithSeparatingPunctuation(text: string) {
-			return separatorChars.some(char => text.startsWith(char))
+			return isWhitespace(char) || isPunctuation(char)
 		}
 
-		function isSeparatorPunctuation(text: string) {
-			return separatorChars.includes(text)
+		function startsWithSeparatorCharacter(text: string) {
+			return isSeparatorCharacter(text[0])
+		}
+
+		function endsWithSeparatorCharacter(text: string) {
+			return isSeparatorCharacter(text[text.length - 1])
 		}
 
 		const resultTimeline: Timeline = []
 
-		const groups: TimelineEntry[][] = []
+		let groups: TimelineEntry[][] = []
 
-		for (let i = 0; i < tokenTimeline.length; i++) {
-			const entry = tokenTimeline[i]
-			const previousEntry = i > 0 ? tokenTimeline[i - 1] : undefined
+		for (let tokenIndex = 0; tokenIndex < tokenTimeline.length; tokenIndex++) {
+			const entry = tokenTimeline[tokenIndex]
+			const previousEntry = tokenIndex > 0 ? tokenTimeline[tokenIndex - 1] : undefined
 
 			const text = entry.text
 			const previousEntryText = previousEntry?.text
 
 			if (groups.length == 0 ||
 				text === '' ||
-				startsWithSeparatingPunctuation(text) ||
-				(previousEntryText != null && isSeparatorPunctuation(previousEntryText))) {
+				startsWithSeparatorCharacter(text) ||
+				(previousEntryText != null && endsWithSeparatorCharacter(previousEntryText))) {
 
 				groups.push([entry])
 			} else {
@@ -1056,8 +1056,28 @@ export class Whisper {
 			}
 		}
 
+		const newGroups: TimelineEntry[][] = []
+
+		for (let groupIndex = 0; groupIndex < groups.length - 1; groupIndex++) {
+			const group = groups[groupIndex]
+			const nextGroup = groups[groupIndex + 1]
+
+			if (
+				group.length > 1 &&
+				group[group.length - 1].text === '.' &&
+				[' ', '['].includes(nextGroup[0].text[0])) {
+
+				newGroups.push(group.slice(0, group.length - 1))
+				newGroups.push(group.slice(group.length - 1))
+			} else {
+				newGroups.push(group)
+			}
+		}
+
+		groups = newGroups
+
 		for (const group of groups) {
-			const groupText = this.tokensToText(group.map(entry => entry.id!))
+			let groupText = this.tokensToText(group.map(entry => entry.id!))
 
 			if (groupText === '') {
 				continue
@@ -1401,6 +1421,90 @@ export class Whisper {
 	getAlignmentHeadIndexes() {
 		return alignmentHeadsIndexes[this.modelName]
 	}
+
+	getSuppressedTokens() {
+		return [
+			...this.getSuppressedTextTokens(),
+			...this.getSuppressedMetadataTokens(),
+		]
+	}
+
+	getSuppressedTextTokens() {
+		const allowedPunctuationMarks = this.getAllowedPunctuationMarks()
+
+		const nonWordTokensData = this.getNonWordTokenData()
+
+		const suppressedTextTokens = nonWordTokensData
+			.filter(entry => !allowedPunctuationMarks.includes(entry.text))
+			.map(entry => entry.id)
+
+		return suppressedTextTokens
+	}
+
+	getSuppressedMetadataTokens() {
+		if (this.isMultiligualModel) {
+			return [50256, ...getIntegerRange(50258, 50364)]
+		} else {
+			return [...getIntegerRange(50257, 50363)]
+		}
+	}
+
+	getAllowedPunctuationMarks() {
+		const generalPunctuation = [`'`, ',', '.', '?', '!']
+
+		let allowedPunctuation: string[]
+
+		if (this.isMultiligualModel) {
+			const spanish = ['¿', '¡']
+			const chinese = ['、', '，', '。', '？', '！']
+			const arabic = ['،', '؟']
+			const various = ['·', '•', '・']
+
+			allowedPunctuation = [...generalPunctuation, ...arabic, ...chinese, ...spanish, ...various]
+		} else {
+			allowedPunctuation = generalPunctuation
+		}
+
+		return allowedPunctuation
+	}
+
+	getNonWordTokenData() {
+		const nonWordTokenData: WhisperTokenData[] = []
+
+		const invalidUTF8Char = String.fromCharCode(65533)
+
+		for (let i = 0; i < this.tokenConfig.endOfTextToken; i++) {
+			const tokenText = this.tokenToText(i, false)
+			const tokenTextWithoutWhitespace = tokenText.replaceAll(/\s/g, '')
+
+			const isNonWordToken = /^[\p{Punctuation}\p{Symbol}]+$/u.test(tokenTextWithoutWhitespace)
+
+			const containsInvalidUTF8 = getUTF32Chars(tokenTextWithoutWhitespace).utf32chars.includes(invalidUTF8Char)
+
+			if (isNonWordToken && !containsInvalidUTF8) {
+				nonWordTokenData.push({
+					id: i,
+					text: tokenText,
+				})
+			}
+		}
+
+		return nonWordTokenData
+	}
+
+
+	getTokensData(tokens: number[]) {
+		const tokensData: WhisperTokenData[] = []
+
+		for (const token of tokens) {
+			tokensData.push({
+				id: token,
+				text: this.tokenToText(token, true),
+			})
+		}
+
+		return tokensData
+	}
 }
 
 const filterbanks: Filterbank[] = [
@@ -1609,6 +1713,11 @@ export function isEnglishOnlyModel(modelName: WhisperModelName) {
 	return modelName.endsWith('.en')
 }
 
+export type WhisperTokenData = {
+	id: number
+	text: string
+}
+
 export type WhisperModelName = 'tiny' | 'tiny.en' | 'base' | 'base.en' | 'small' | 'small.en' | 'medium' | 'medium.en' | 'large' | 'large-v1' | 'large-v2' | 'large-v3'
 export type WhisperTask = 'transcribe' | 'translate' | 'detect-language'
 
@@ -1755,8 +1864,8 @@ export interface WhisperOptions {
 	autoPromptParts?: boolean
 	maxTokensPerPart?: number
 	suppressRepetition?: boolean
-	seed?: number
 	decodeTimestampTokens?: boolean
+	seed?: number
 }
 
 export const defaultWhisperOptions: WhisperOptions = {
@@ -1768,6 +1877,6 @@ export const defaultWhisperOptions: WhisperOptions = {
 	autoPromptParts: true,
 	maxTokensPerPart: 250,
 	suppressRepetition: true,
+	decodeTimestampTokens: true,
 	seed: undefined,
-	decodeTimestampTokens: false,
 }

package/src/source-separation/MDXNetSourceSeparation.ts

@@ -1,4 +1,4 @@
-import Onnx from 'onnxruntime-node'
+import type * as Onnx from 'onnxruntime-node'
 import { RawAudio } from '../audio/AudioUtilities.js';
 import { binBufferToComplex, complexToBinBuffer, getWindowWeights, stftr, stiftr } from '../dsp/FFT.js';
 import { ComplexNumber } from '../math/VectorMath.js';
@@ -32,6 +32,8 @@ export class MDXNet {
 			await this.initializeSession(this.modelFilePath)
 		}
 
+		const Onnx = await import('onnxruntime-node')
+
 		const logger = new Logger()
 
 		const session = this.session!
@@ -223,6 +225,8 @@ export class MDXNet {
 			logSeverityLevel: 3
 		}
 
+		const Onnx = await import('onnxruntime-node')
+
 		this.session = await Onnx.InferenceSession.create(modelPath, onnxOptions)
 	}
 }

package/src/speech-language-detection/SileroLanguageDetection.ts

@@ -1,4 +1,4 @@
-import Onnx from 'onnxruntime-node'
+import type * as Onnx from 'onnxruntime-node'
 import { softmax } from '../math/VectorMath.js'
 import { Logger } from '../utilities/Logger.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
@@ -50,6 +50,8 @@ export class SileroLanguageDetection {
 			logSeverityLevel: 3
 		}
 
+		const Onnx = await import('onnxruntime-node')
+
 		this.session = await Onnx.InferenceSession.create(this.modelPath, onnxOptions)
 
 		logger.end()
@@ -62,6 +64,8 @@ export class SileroLanguageDetection {
 
 		const audioSamples = rawAudio.audioChannels[0]
 
+		const Onnx = await import('onnxruntime-node')
+		
 		const inputTensor = new Onnx.Tensor('float32', audioSamples, [1, audioSamples.length])
 
 		const inputs = { input: inputTensor }

package/src/subtitles/Subtitles.ts

@@ -200,7 +200,7 @@ function getCuesFromTimeline_IsolateSegmentSentence(timeline: Timeline, config:
 			continue
 		}
 
-		const wordTimeline = entry.timeline!.filter(entry => isWord(entry.text))
+		const wordTimeline = entry.timeline!.filter(entry => isWordOrSymbolWord(entry.text))
 
 		// First, add word start and end offsets for all word entries
 		let lastWordEndOffset = 0

package/src/synthesis/VitsTTS.ts

@@ -1,4 +1,4 @@
-import type { InferenceSession } from 'onnxruntime-node'
+import type * as Onnx from 'onnxruntime-node'
 import { SynthesisVoice } from '../api/API.js'
 import { Logger } from '../utilities/Logger.js'
 import { RawAudio, getEmptyRawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
@@ -32,7 +32,7 @@ export class VitsTTS {
 	voiceName: string
 	modelPath: string
 
-	modelSession?: InferenceSession
+	modelSession?: Onnx.InferenceSession
 	metadata?: any
 	phonemeMap?: Map<string, number[]>
 
@@ -45,7 +45,7 @@ export class VitsTTS {
 		const logger = new Logger()
 		await logger.startAsync('Initialize VITS ONNX synthesis model')
 
-		const onnxOptions: InferenceSession.SessionOptions = {
+		const onnxOptions: Onnx.InferenceSession.SessionOptions = {
 			logSeverityLevel: 3
 		}
 

package/src/tests/Test.ts

@@ -1,19 +1,17 @@
-import { getRepetitionScoreRelativeToFirstSubstring, logToStderr, setupProgramTerminationListeners } from '../utilities/Utilities.js'
+import { getRepetitionScoreRelativeToFirstSubstring, logToStderr, setupProgramTerminationListeners, writeToStderr } from '../utilities/Utilities.js'
 import { makeTarballsForInstalledPackages } from '../utilities/TarballMaker.js'
 import { testEspeakSynthesisWithPrePhonemizedInputs, testKirshenbaumPhonemization } from '../synthesis/EspeakTTS.js'
 
 const log = logToStderr
 
 setupProgramTerminationListeners()
+
 //process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
 //process.env.http_proxy = 'http://localhost:8080'
 
 //const testText = `After a while, finding that nothing more happened, she decided on going into the garden at once; but, alas for poor Alice! when she got to the door, she found she had forgotten the little golden key, and when she went back to the table for it, she found she could not possibly reach it: she could see it quite plainly through the glass, and she tried her best to climb up one of the legs of the table, but it was too slippery; and when she had tired herself out with trying, the poor little thing sat down and cried.`
-
 //await testKirshenbaumPhonemization(testText)
 
-//await makeTarballsForInstalledPackages(true)
-
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'c', 'a', 'b', 'c'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'd', 'a', 'b', 'a', 'd'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'b', 'c', 'a', 'b', 'a', 'b'])
@@ -21,4 +19,7 @@ setupProgramTerminationListeners()
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'c', 'a', 'b', 'a', 'c', 'a'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'b', 'a', 'a', 'a', 'b'])
 
+//await makeTarballsForInstalledPackages(true)
+
 process.exit(0)
+

package/src/voice-activity-detection/SileroVAD.ts

@@ -1,4 +1,4 @@
-import Onnx from 'onnxruntime-node'
+import type * as Onnx from 'onnxruntime-node'
 
 import { concatFloat32Arrays } from '../utilities/Utilities.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
@@ -38,33 +38,24 @@ export async function detectVoiceActivity(rawAudio: RawAudio, modelPath: string,
 export class SileroVAD {
 	session?: Onnx.InferenceSession
 
-	modelStateH: Onnx.Tensor
-	modelStateC: Onnx.Tensor
-	modelSampleRate: Onnx.Tensor
+	modelStateH?: Onnx.Tensor
+	modelStateC?: Onnx.Tensor
+	modelSampleRate?: Onnx.Tensor
 
 	modelPath: string
 
 	constructor(modelPath: string) {
-		this.modelSampleRate = new Onnx.Tensor('int64', new BigInt64Array([BigInt(16000)]), [])
-
-		const h = new Float32Array(2 * 1 * 64)
-		const c = new Float32Array(2 * 1 * 64)
-
-		this.modelStateH = new Onnx.Tensor('float32', h, [2, 1, 64])
-		this.modelStateC = new Onnx.Tensor('float32', c, [2, 1, 64])
-
 		this.modelPath = modelPath
 	}
 
 	async predictAudioFrame(frame: Float32Array) {
-		if (!this.session) {
-			await this.initializeSession(this.modelPath)
-		}
+		await this.initializeIfNeeded()
+
+		const Onnx = await import('onnxruntime-node')
 
 		const inputTensor = new Onnx.Tensor('float32', frame, [1, frame.length])
 
-		//const inputs = { input: inputTensor, h0: this.modelStateH, c0: this.modelStateC }
-		const inputs = { input: inputTensor, sr: this.modelSampleRate, h: this.modelStateH, c: this.modelStateC }
+		const inputs = { input: inputTensor, sr: this.modelSampleRate!, h: this.modelStateH!, c: this.modelStateC! }
 
 		const results = await this.session!.run(inputs)
 
@@ -75,12 +66,25 @@ export class SileroVAD {
 
 		return probability
 	}
+	private async initializeIfNeeded() {
+		if (this.session) {
+			return
+		}
+
+		const Onnx = await import('onnxruntime-node')
+
+		const h = new Float32Array(2 * 1 * 64)
+		const c = new Float32Array(2 * 1 * 64)
+
+		this.modelStateH = new Onnx.Tensor('float32', h, [2, 1, 64])
+		this.modelStateC = new Onnx.Tensor('float32', c, [2, 1, 64])
+
+		this.modelSampleRate = new Onnx.Tensor('int64', new BigInt64Array([BigInt(16000)]), [])
 
-	private async initializeSession(modelPath: string) {
 		const onnxOptions: Onnx.InferenceSession.SessionOptions = {
 			logSeverityLevel: 3
 		}
 
-		this.session = await Onnx.InferenceSession.create(modelPath, onnxOptions)
+		this.session = await Onnx.InferenceSession.create(this.modelPath, onnxOptions)
 	}
 }