echogarden 2.5.2 → 2.7.0

package/src/audio/AudioRecorder.ts

@@ -6,8 +6,8 @@ import * as AudioBufferConversion from './AudioBufferConversion.js'
 import { Timer } from '../utilities/Timer.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
-import { SampleFormat } from '../codecs/WaveCodec.js'
 import { tryResolvingSoxPath } from './SoxPath.js'
+import { SampleFormat } from './AudioBufferConversion.js'
 
 const log = logToStderr
 

package/src/audio/AudioUtilities.ts

@@ -1,5 +1,5 @@
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
-import { SampleFormat, encodeWave, decodeWave, BitDepth } from '../codecs/WaveCodec.js'
+import { SampleFormat, BitDepth, encodeWaveFromFloat32Channels, decodeWaveToFloat32Channels } from '@echogarden/wave-codec'
 import { resampleAudioSpeex } from '../dsp/SpeexResampler.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { concatFloat32Arrays } from '../utilities/Utilities.js'
@@ -8,11 +8,13 @@ import { concatFloat32Arrays } from '../utilities/Utilities.js'
 // Wave encoding and decoding
 ////////////////////////////////////////////////////////////////////////////////////////////////
 export function encodeRawAudioToWave(rawAudio: RawAudio, bitDepth: BitDepth = 16, sampleFormat: SampleFormat = SampleFormat.PCM, speakerPositionMask = 0) {
-	return encodeWave(rawAudio, bitDepth, sampleFormat, speakerPositionMask)
+	return encodeWaveFromFloat32Channels(rawAudio.audioChannels, rawAudio.sampleRate, bitDepth, sampleFormat, speakerPositionMask)
 }
 
 export function decodeWaveToRawAudio(waveFileBuffer: Uint8Array, ignoreTruncatedChunks = true, ignoreOverflowingDataChunks = true) {
-	return decodeWave(waveFileBuffer, ignoreTruncatedChunks, ignoreOverflowingDataChunks)
+	const rawAudio = decodeWaveToFloat32Channels(waveFileBuffer, ignoreTruncatedChunks, ignoreOverflowingDataChunks)
+
+	return { rawAudio }
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////

package/src/codecs/TIMITCodec.ts

@@ -1,7 +1,6 @@
-import { decodeToChannels } from '../audio/AudioBufferConversion.js'
+import { decodeToChannels, SampleFormat } from '../audio/AudioBufferConversion.js'
 import { decodeAscii } from '../encodings/Ascii.js'
 import { readFileAsBinary } from '../utilities/FileSystem.js'
-import { SampleFormat } from './WaveCodec.js'
 
 export async function decodeTimitAudioFile(filename: string) {
 	return decodeTimitAudio(await readFileAsBinary(filename))

package/src/encodings/Ascii.ts

@@ -49,7 +49,7 @@ export class ChunkedAsciiDecoder {
 	private readonly textDecoder = new TextDecoder('windows-1252')
 
 	writeChunk(chunk: Uint8Array) {
-		const decodedChunk = this.textDecoder.decode(chunk)
+		const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
 
 		this.str += decodedChunk
 	}

package/src/encodings/Utf16.ts

@@ -41,7 +41,7 @@ export class ChunkedUtf16Decoder {
 	private readonly textDecoder = new TextDecoder('utf-16le')
 
 	writeChunk(chunk: Uint16Array) {
-		const decodedChunk = this.textDecoder.decode(chunk)
+		const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
 
 		this.str += decodedChunk
 	}

package/src/encodings/Utf8.ts

@@ -33,7 +33,7 @@ export class ChunkedUtf8Decoder {
 	private readonly textDecoder = new TextDecoder('utf-8')
 
 	writeChunk(chunk: Uint8Array) {
-		const decodedChunk = this.textDecoder.decode(chunk)
+		const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
 
 		this.str += decodedChunk
 	}

package/src/math/VectorMath.ts

@@ -727,6 +727,8 @@ export function sigmoid(x: number) {
 }
 
 export function softmax(logits: ArrayLike<number>, temperature = 1.0) {
+	temperature = Math.max(temperature, 0.00001)
+
 	const logitCount = logits.length
 
 	if (logitCount === 0) {
@@ -743,23 +745,23 @@ export function softmax(logits: ArrayLike<number>, temperature = 1.0) {
 		}
 	}
 
-	const temperatureReciprocal = 1 / (temperature + 1e-40)
+	const temperatureReciprocal = 1 / temperature
 
 	const results = new Float32Array(logitCount)
 
-	let sumOfExponentiatedValues = 0.0
+	let sumOfExponentiatedLogits = 0.0
 
 	for (let i = 0; i < logitCount; i++) {
-		const value = logits[i]
+		const logit = logits[i]
 
-		const eToValue = Math.exp((value - maxValue) * temperatureReciprocal)
+		const eToLogit = Math.exp((logit - maxValue) * temperatureReciprocal)
 
-		sumOfExponentiatedValues += eToValue
+		sumOfExponentiatedLogits += eToLogit
 
-		results[i] = eToValue
+		results[i] = eToLogit
 	}
 
-	const sumOfExponentiatedValuesReciprocal = 1 / (sumOfExponentiatedValues + 1e-40)
+	const sumOfExponentiatedValuesReciprocal = 1 / (sumOfExponentiatedLogits + 1e-20)
 
 	for (let i = 0; i < logitCount; i++) {
 		results[i] *= sumOfExponentiatedValuesReciprocal

package/src/nlp/CompromiseNLP.ts

@@ -79,12 +79,16 @@ export function tryMatchInLexicon(term: CompromiseParsedTerm, lexicon: Lexicon,
 	const termText = term.text
 	const lowerCaseTermText = termText.toLocaleLowerCase()
 
-	const entry = lexiconForLanguage[lowerCaseTermText]
+	let entry = lexiconForLanguage[lowerCaseTermText]
 
 	if (!entry) {
 		return undefined
 	}
 
+	if (!Array.isArray(entry)) {
+		entry = [entry]
+	}
+
 	for (const substitutionEntry of entry) {
 		if (!substitutionEntry.pos || substitutionEntry.pos.includes(term.pos)) {
 			const substitutionPhonemesText = substitutionEntry?.pronunciation?.espeak?.[espeakVoice]

package/src/nlp/Lexicon.ts

@@ -30,12 +30,16 @@ export function tryGetLexiconSubstitution(sentenceWords: string[], wordIndex: nu
 		return undefined
 	}
 
-	const lexiconEntry = lexiconForLanguage[word]
+	let lexiconEntry = lexiconForLanguage[word]
 
 	if (!lexiconEntry) {
 		return undefined
 	}
 
+	if (!Array.isArray(lexiconEntry)) {
+		lexiconEntry = [lexiconEntry]
+	}
+
 	for (let i = 0; i < lexiconEntry.length; i++) {
 		const candidateEntry = lexiconEntry[i]
 
@@ -100,7 +104,7 @@ export type Lexicon = {
 }
 
 export type LexiconForLanguage = {
-	[word: string]: LexiconEntry[]
+	[word: string]: LexiconEntry | LexiconEntry[]
 }
 
 export type LexiconEntry = {

package/src/nlp/PhoneConversion.ts

@@ -1,14 +1,26 @@
 export function ipaPhoneToKirshenbaum(ipaPhone: string) {
 	let result = ''
 
-	for (const char of ipaPhone) {
-		const convertedChar = ipaToKirshenbaum[char]
+	if (ipaPhone[0] === 'ˈ' || ipaPhone[0] === 'ˌ') {
+		result += ipaToKirshenbaum[ipaPhone[0]]
 
-		if (convertedChar == undefined) {
-			throw new Error(`Couldn't convert IPA character '${char}' (part of phone '${ipaPhone}') to the Kirshenbaum notation`)
-		}
+		ipaPhone = ipaPhone.substring(1)
+	}
+
+	const convertedFullPhone = ipaToKirshenbaum[ipaPhone]
 
-		result += convertedChar || '_'
+	if (convertedFullPhone) {
+		result += convertedFullPhone
+	} else {
+		for (const char of ipaPhone) {
+			const convertedChar = ipaToKirshenbaum[char]
+
+			if (convertedChar === undefined) {
+				throw new Error(`Couldn't convert IPA character '${char}' (part of phone '${ipaPhone}') to the Kirshenbaum notation`)
+			}
+
+			result += convertedChar ?? '_'
+		}
 	}
 
 	return result
@@ -252,6 +264,8 @@ const ipaToTimit: { [p: string]: string[] | undefined } = {
 	'ɾ': ['dx'],
 }
 
+// Kirshenbaum is an ASCII encoding of IPA
+// The variant used by eSpeak is used here, and has many differences to standard Kirshenbaum
 export const ipaToKirshenbaum: Record<string, string> = {
 	//// Vowels
 

package/src/recognition/WhisperCppSTT.ts

@@ -405,7 +405,7 @@ export type WhisperCppBuild = 'cpu' | 'cublas-12.4.0' | 'custom'
 
 export async function loadExecutablePackage(buildKind: WhisperCppBuild) {
 	if (buildKind === 'custom') {
-		throw new Error(`A 'custom' build kind requires providing a custom path to the whisper.cpp binary in the 'executablePath' option.`)
+		throw new Error(`A 'custom' build kind requires providing a custom path to the 'whisper-cli' executable in the 'executablePath' option.`)
 	}
 
 	const platform = process.platform
@@ -416,8 +416,10 @@ export async function loadExecutablePackage(buildKind: WhisperCppBuild) {
 	if (buildKind.startsWith('cublas-')) {
 		if (platform === 'win32' && arch === 'x64') {
 			packageName = `whisper.cpp-binaries-windows-x64-${buildKind}-latest`
+		} else if (platform === 'linux' && arch === 'x64') {
+			packageName = `whisper.cpp-binaries-linux-x64-${buildKind}-latest`
 		} else {
-			throw new Error(`whisper.cpp GPU builds (NVIDIA CUDA only) are currently only available as packages for Windows x64. Please specify a custom path to a whisper.cpp 'main' binary in the 'executablePath' option.`)
+			throw new Error(`whisper.cpp GPU builds (NVIDIA CUDA only) are currently only available as packages for Windows x64 and Linux x64. Please specify a custom path to a whisper.cpp 'main' binary in the 'executablePath' option.`)
 		}
 	} else if (buildKind === 'cpu') {
 		if (platform === 'win32' && arch === 'x64') {
@@ -433,7 +435,7 @@ export async function loadExecutablePackage(buildKind: WhisperCppBuild) {
 
 	const packagePath = await loadPackage(packageName)
 
-	let filename = 'main'
+	let filename = 'whisper-cli' // used to be called 'main' but 'main' is now deprecated
 
 	if (platform === 'win32') {
 		filename += '.exe'

package/src/recognition/WhisperSTT.ts

@@ -2,8 +2,8 @@ import type * as Onnx from 'onnxruntime-node'
 
 import { Logger } from '../utilities/Logger.js'
 import { computeMelSpectogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectogram.js'
-import { clip, concatFloat32Arrays, getIntegerRange, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
-import { indexOfMax, logOfVector, logSumExp, meanOfVector, softmax, stdDeviationOfVector, sumOfSquaresForVector, sumVector } from '../math/VectorMath.js'
+import { clip, getIntegerRange, getTopKIndexes, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
+import { indexOfMax, logOfVector, logSumExp, meanOfVector, softmax, sumOfSquaresForVector, sumVector } from '../math/VectorMath.js'
 
 import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
@@ -1019,36 +1019,37 @@ export class Whisper {
 			}
 
 			// Find top candidates
-			const sortedNonTimestampLogitsWithIndexes =
-				Array.from(nonTimestampTokenLogits).map((logit, index) => ({ token: index, logit }))
+			const sortedTopCandidateTokens = getTopKIndexes(nonTimestampTokenLogits, options.topCandidateCount!, true)
 
-			sortedNonTimestampLogitsWithIndexes.sort((a, b) => b.logit - a.logit)
+			let topCandidates = Array.from(sortedTopCandidateTokens).map(token => {
+				const logit = nonTimestampTokenLogits[token]
 
-			let topCandidates = sortedNonTimestampLogitsWithIndexes.slice(0, options.topCandidateCount!)
-				.map(entry => ({
-					token: entry.token,
-					logit: entry.logit,
-					text: this.tokenToText(entry.token, true)
-				}))
+				return {
+					token,
+					logit,
+					text: this.tokenToText(token, true)
+				}
+			})
 
 			// Apply repetition suppression if enabled
 			if (options.suppressRepetition) {
-				// Using some hardcoded constants, for now
+				// Using some hardcoded constants, for now:
 				const tokenWindowSize = 30
 				const thresholdMatchLength = 4
-				const thresholdCycleRepetition = 3
+				const thresholdCycleRepetitionCount = 3
 
 				const filteredCandidates: typeof topCandidates = []
 
 				for (const candidate of topCandidates) {
-					const lastDecodedTextTokens = decodedTokens
-						.filter(token => this.isTextToken(token))
+					const decodedTextTokens = decodedTokens.filter(token => this.isTextToken(token))
+
+					const lastDecodedTextTokens = decodedTextTokens
+						.slice(Math.max(decodedTextTokens.length - tokenWindowSize, 0))
 						.reverse()
-						.slice(0, tokenWindowSize)
 
-					const { longestMatch, longestCycleRepetition } = getTokenRepetitionScore([candidate.token, ...lastDecodedTextTokens])
+					const { longestMatchLength, longestCycleRepetitionCount } = getTokenRepetitionScore([candidate.token, ...lastDecodedTextTokens])
 
-					if (longestMatch >= thresholdMatchLength || longestCycleRepetition >= thresholdCycleRepetition) {
+					if (longestMatchLength >= thresholdMatchLength || longestCycleRepetitionCount >= thresholdCycleRepetitionCount) {
 						continue
 					}
 

package/src/synthesis/SapiTTS.ts

@@ -1,7 +1,6 @@
 import { SynthesisVoice } from '../api/API.js'
-import { decodeToChannels } from '../audio/AudioBufferConversion.js'
+import { decodeToChannels, SampleFormat } from '../audio/AudioBufferConversion.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
-import { SampleFormat } from '../codecs/WaveCodec.js'
 import { getShortLanguageCode, lcidToIsoLanguageCode } from '../utilities/Locale.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'

package/src/synthesis/SvoxPicoTTS.ts

@@ -1,6 +1,5 @@
 import { SynthesisVoice } from '../api/API.js'
-import { decodeToChannels } from '../audio/AudioBufferConversion.js'
-import { SampleFormat } from '../codecs/WaveCodec.js'
+import { decodeToChannels, SampleFormat } from '../audio/AudioBufferConversion.js'
 import { bandwidthToQFactor } from '../dsp/BiquadFilter.js'
 import { Logger } from '../utilities/Logger.js'
 import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js'

package/src/utilities/PackageManager.ts

@@ -197,9 +197,10 @@ const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
 	'whisper.cpp-large-v3-turbo-q5_0': '20241003',
 
 	// whisper.cpp binaries
-	'whisper.cpp-binaries-linux-x64-cpu-latest': '20241010',
-	'whisper.cpp-binaries-windows-x64-cpu-latest': '20241010',
-	'whisper.cpp-binaries-windows-x64-cublas-12.4.0-latest': '20241010',
+	'whisper.cpp-binaries-windows-x64-cpu-latest': '20250502',
+	'whisper.cpp-binaries-windows-x64-cublas-12.4.0-latest': '20250502',
+	'whisper.cpp-binaries-linux-x64-cpu-latest': '20250502',
+	'whisper.cpp-binaries-linux-x64-cublas-12.4.0-latest': '20250502',
 
 	// E5 models
 	'xenova-multilingual-e5-small-q8': '20240504',

package/src/utilities/StringUtilities.ts

@@ -148,24 +148,24 @@ export function getTokenRepetitionScore(tokens: string[] | number[]) {
 		matchLengthForCycleLength.push(score)
 	}
 
-	let longestMatch = -Infinity
-	let longestCycleRepetition = -Infinity
+	let longestMatchLength = -Infinity
+	let longestCycleRepetitionCount = -Infinity
 
 	for (let i = 1; i <= matchLengthForCycleLength.length; i++) {
 		const matchLength = matchLengthForCycleLength[i]
 
-		if (matchLength > longestMatch) {
-			longestMatch = matchLength
+		if (matchLength > longestMatchLength) {
+			longestMatchLength = matchLength
 		}
 
 		const cycleCount = (matchLength / i) + 1
 
-		if (cycleCount > longestCycleRepetition) {
-			longestCycleRepetition = cycleCount
+		if (cycleCount > longestCycleRepetitionCount) {
+			longestCycleRepetitionCount = cycleCount
 		}
 	}
 
-	return { longestMatch, longestCycleRepetition }
+	return { longestMatchLength, longestCycleRepetitionCount }
 }
 
 export async function convertHtmlToText(html: string) {

package/src/utilities/Utilities.ts

@@ -511,3 +511,67 @@ export function encodeHTMLAngleBrackets(text: string) {
 		.replaceAll('>', '>')
 		.replaceAll('&', '&')
 }
+
+export function getTopKIndexes(values: ArrayLike<number>, topCount: number, sort = true) {
+	if (topCount < 1) {
+		throw new Error(`Top count must be at least 1`)
+	}
+
+	const topKIndexes = new Uint32Array(Math.min(topCount, values.length))
+
+	// Initialize top k indexes with the first k elements
+	// (or less, if value list is shorter)
+	for (let i = 0; i < topKIndexes.length; i++) {
+		topKIndexes[i] = i
+	}
+
+	// Return if value list is shorter or equal in length to topCount
+	if (values.length <= topCount) {
+		return topKIndexes
+	}
+
+	////
+
+	let positionOfMinimum = -1
+	let valueOfMinimum = -Infinity
+
+	// Method to scan the top-k array and update the latest minimum value position and value
+	function updateMinimum() {
+		positionOfMinimum = 0
+		valueOfMinimum = values[topKIndexes[0]]
+
+		for (let i = 1; i < topKIndexes.length; i++) {
+			const value = values[topKIndexes[i]]
+
+			if (value < valueOfMinimum) {
+				positionOfMinimum = i
+				valueOfMinimum = value
+			}
+		}
+	}
+
+	updateMinimum()
+
+	// Add remaining elements to the list, if needed
+	for (let insertedElementIndex = topCount; insertedElementIndex < values.length; insertedElementIndex++) {
+		const insertedElementValue = values[insertedElementIndex]
+
+		// If the inserted element's value is lesser or equal to the value
+		// of the smallest value on the list, skip it
+		if (insertedElementValue <= valueOfMinimum) {
+			continue
+		}
+
+		// Replace the minimum element with the inserted element
+		topKIndexes[positionOfMinimum] = insertedElementIndex
+
+		// Update the position and value of the minimum element
+		updateMinimum()
+	}
+
+	if (sort) {
+		topKIndexes.sort((a, b) => values[b] - values[a])
+	}
+
+	return topKIndexes
+}

package/dist/codecs/WaveCodec.d.ts

@@ -1,19 +0,0 @@
-import { RawAudio } from '../audio/AudioUtilities.js';
-export declare function encodeWave(rawAudio: RawAudio, bitDepth?: BitDepth, sampleFormat?: SampleFormat, speakerPositionMask?: number): Uint8Array<ArrayBufferLike>;
-export declare function decodeWave(waveData: Uint8Array, ignoreTruncatedChunks?: boolean, ignoreOverflowingDataChunks?: boolean): {
-    rawAudio: {
-        audioChannels: Float32Array<ArrayBufferLike>[];
-        sampleRate: number;
-    };
-    sourceSampleFormat: SampleFormat;
-    sourceBitDepth: BitDepth;
-    sourceSpeakerPositionMask: number;
-};
-export declare function repairWave(waveData: Uint8Array): Uint8Array<ArrayBufferLike>;
-export declare enum SampleFormat {
-    PCM = 1,
-    Float = 3,
-    Alaw = 6,
-    Mulaw = 7
-}
-export type BitDepth = 8 | 16 | 24 | 32 | 64;