echogarden 2.10.1 → 3.0.0

package/src/utilities/PackageManager.ts

@@ -1,11 +1,11 @@
-import { downloadAndExtractTarball } from './FileDownloader.js'
+import { downloadAndExtractTarball, FileDownloaderCallbacks } from './FileDownloader.js'
 import { getAppDataDir, ensureDir, existsSync, remove } from './FileSystem.js'
 import { appName } from '../api/Common.js'
-import { GaxiosOptions } from 'gaxios'
 import { getAppTempDir, joinPath } from './PathUtilities.js'
 import { getGlobalOption } from '../api/GlobalOptions.js'
+import { EasierHttpRequestConfig } from 'easier-http-request'
 
-export async function loadPackage(packageName: string) {
+export async function loadPackage(packageName: string, callbacks: LoadPackageCallbacks) {
 	packageName = resolveToVersionedPackageNameIfNeeded(packageName)
 
 	const packagesPath = await ensureAndGetPackagesDir()
@@ -23,16 +23,18 @@ export async function loadPackage(packageName: string) {
 	const headers = {
 	}
 
-	const options: GaxiosOptions = {
+	const config: EasierHttpRequestConfig = {
 		url: `${packageBaseURL}${packageName}.tar.gz`,
-		headers
+		headers,
+		abortSignal: callbacks?.abortSignal
 	}
 
 	await downloadAndExtractTarball(
-		options,
+		config,
 		packagesPath,
 		tempPath,
 		packageName,
+		callbacks,
 	)
 
 	return packagePath
@@ -81,8 +83,8 @@ export function resolveVersionTagForUnversionedPackageName(unversionedPackageNam
 const defaultVersionTag = '20230718'
 
 const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
-	// SOX binary
-	'sox-14.4.2-linux-minimal': '20230802',
+	// eSpeak-NG
+	'espeak-ng-emscripten': '20260722',
 
 	// VITS voices
 	'vits-de_DE-thorsten_emotional-medium': '20230808',
@@ -131,17 +133,6 @@ const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
 	'kokoro-82m-v1.0-quantized': '20250209',
 	'kokoro-82m-v1.0-voices': '20250209',
 
-	// Whisper (integrated engine) models
-	'whisper-tiny': '20231126',
-	'whisper-tiny.en': '20231126',
-	'whisper-base': '20231126',
-	'whisper-base.en': '20231126',
-	'whisper-small': '20231126',
-	'whisper-small.en': '20231126',
-	'whisper-medium': '20231126',
-	'whisper-medium.en': '20231126',
-	'whisper-large-v3-turbo-fp16': '20241002',
-
 	// Whisper tiktoken data
 	'whisper-tiktoken-data': '20240408',
 
@@ -169,38 +160,71 @@ const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
 	// whisper.cpp models
 	'whisper.cpp-tiny': '20240405',
 	'whisper.cpp-tiny-q5_1': '20240405',
+	'whisper.cpp-tiny-q8_0': '20260725',
+
 	'whisper.cpp-tiny.en': '20240405',
 	'whisper.cpp-tiny.en-q5_1': '20240405',
-	'whisper.cpp-tiny.en-q8_0': '20240405',
+	'whisper.cpp-tiny.en-q8_0': '20260725',
 
 	'whisper.cpp-base': '20240405',
 	'whisper.cpp-base-q5_1': '20240405',
+	'whisper.cpp-base-q8_0': '20260725',
+
 	'whisper.cpp-base.en': '20240405',
 	'whisper.cpp-base.en-q5_1': '20240405',
+	'whisper.cpp-base.en-q8_0': '20260725',
 
 	'whisper.cpp-small': '20240405',
 	'whisper.cpp-small-q5_1': '20240405',
+	'whisper.cpp-small-q8_0': '20260725',
+
 	'whisper.cpp-small.en': '20240405',
 	'whisper.cpp-small.en-q5_1': '20240405',
+	'whisper.cpp-small.en-q8_0': '20260725',
 
 	'whisper.cpp-medium': '20240405',
 	'whisper.cpp-medium-q5_0': '20240405',
+	'whisper.cpp-medium-q8_0': '20260725',
+
 	'whisper.cpp-medium.en': '20240405',
 	'whisper.cpp-medium.en-q5_0': '20240405',
+	'whisper.cpp-medium.en-q8_0': '20260725',
 
 	'whisper.cpp-large-v1': '20240405',
+
 	'whisper.cpp-large-v2': '20240405',
 	'whisper.cpp-large-v2-q5_0': '20240405',
+	'whisper.cpp-large-v2-q8_0': '20260725',
+
 	'whisper.cpp-large-v3': '20240405',
 	'whisper.cpp-large-v3-q5_0': '20240405',
+
 	'whisper.cpp-large-v3-turbo': '20241003',
 	'whisper.cpp-large-v3-turbo-q5_0': '20241003',
-
-	// whisper.cpp binaries
-	'whisper.cpp-binaries-windows-x64-cpu-latest': '20250502',
-	'whisper.cpp-binaries-windows-x64-cublas-12.4.0-latest': '20250502',
-	'whisper.cpp-binaries-linux-x64-cpu-latest': '20250502',
-	'whisper.cpp-binaries-linux-x64-cublas-12.4.0-latest': '20250502',
+	'whisper.cpp-large-v3-turbo-q8_0': '20260725',
+
+	// whisper.cpp library fork used for binding (introduced in Echogarden v3.0.0)
+	'whisper.cpp-lib-windows-x64-cpu': '20260720',
+	'whisper.cpp-lib-windows-x64-cublas-12.4.0': '20260720',
+	'whisper.cpp-lib-windows-arm64-cpu': '20260720',
+	'whisper.cpp-lib-macos-universal': '20260720',
+	'whisper.cpp-lib-macos-universal-cpu-basic': '20260720',
+	'whisper.cpp-lib-linux-x64-cpu': '20260720',
+	'whisper.cpp-lib-linux-x64-cublas-12.4.0': '20260720',
+	'whisper.cpp-lib-linux-arm64-cpu': '20260720',
+
+	// CuBLAS dynamically linked libraries Windows x64 distribution
+	'cublas-lib-12.4.0-windows-x64': '20250607',
+
+	// whisper.cpp CLI binaries
+	'whisper.cpp-binaries-windows-x64-cpu': '20260720',
+	'whisper.cpp-binaries-windows-x64-cublas-12.4.0': '20260720',
+	'whisper.cpp-binaries-windows-arm64-cpu': '20260720',
+	'whisper.cpp-binaries-macos-universal': '20260720',
+	'whisper.cpp-binaries-macos-universal-cpu-basic': '20260720',
+	'whisper.cpp-binaries-linux-x64-cpu': '20260720',
+	'whisper.cpp-binaries-linux-x64-cublas-12.4.0': '20260720',
+	'whisper.cpp-binaries-linux-arm64-cpu': '20260720',
 
 	// E5 models
 	'xenova-multilingual-e5-small-q8': '20240504',
@@ -220,3 +244,6 @@ const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
 	'nsnet2-20ms-baseline': '20242610',
 	'nsnet2-20ms-48k-baseline': '20242610',
 }
+
+export interface LoadPackageCallbacks extends FileDownloaderCallbacks {
+}

package/src/utilities/PathUtilities.ts

@@ -1,9 +1,9 @@
 import * as NodePath from 'node:path'
-import { fileURLToPath } from 'node:url'
+import * as NodeUrl from 'node:url'
 import * as os from 'node:os'
 
 export function getModuleRootDir() {
-	const currentScriptDir = getDirName(fileURLToPath(import.meta.url))
+	const currentScriptDir = getDirName(NodeUrl.fileURLToPath(import.meta.url))
 
 	return resolvePath(currentScriptDir, '..', '..')
 }
@@ -53,6 +53,10 @@ export function normalizePath(path: string) {
 	return NodePath.normalize(path)
 }
 
+export function pathToFileUrl(path: string) {
+	return NodeUrl.pathToFileURL(path)
+}
+
 export function getBaseName(path: string) {
 	return NodePath.basename(path)
 }

package/src/utilities/TarballMaker.ts

@@ -1,22 +1,22 @@
 import { createTarball } from './Compression.js'
 import { ensureDir, move, readdir, stat } from './FileSystem.js'
-import { appName } from '../api/Common.js'
+import { appName, OperationCallbacks } from '../api/Common.js'
 import { getRandomHexString } from './Utilities.js'
 import { getAppTempDir, getFileNameWithoutExtension, joinPath } from './PathUtilities.js'
 
-export async function createNamedTarball(inputPath: string, name: string, tarballDir: string) {
+export async function createNamedTarball(inputPath: string, name: string, tarballDir: string, callbacks: OperationCallbacks) {
 	const tempDir = getAppTempDir(appName)
 	await ensureDir(tempDir)
 	const tempFilename = joinPath(tempDir, getRandomHexString(16))
 
-	await createTarball(inputPath, tempFilename, name)
+	await createTarball(inputPath, tempFilename, name, callbacks)
 
 	const targetFilname = joinPath(tarballDir, `${name}.tar.gz`)
 
 	await move(tempFilename, targetFilname)
 }
 
-export async function createTarballForEachDirIn(baseDir: string, namePrefix: string, tarballDir: string) {
+export async function createTarballForEachDirIn(baseDir: string, namePrefix: string, tarballDir: string, callbacks: OperationCallbacks) {
 	for (const dirName of await readdir(baseDir)) {
 		const dirPath = joinPath(baseDir, dirName)
 
@@ -28,11 +28,11 @@ export async function createTarballForEachDirIn(baseDir: string, namePrefix: str
 
 		const archiveName = `${namePrefix}-${dirName}`
 
-		await createNamedTarball(dirPath, archiveName, tarballDir)
+		await createNamedTarball(dirPath, archiveName, tarballDir, callbacks)
 	}
 }
 
-export async function createTarballForEachFileIn(baseDir: string, namePrefix: string, tarballDir: string) {
+export async function createTarballForEachFileIn(baseDir: string, namePrefix: string, tarballDir: string, callbacks: OperationCallbacks) {
 	for (const filename of await readdir(baseDir)) {
 		const filenameWithoutExtension = getFileNameWithoutExtension(filename)
 
@@ -46,6 +46,6 @@ export async function createTarballForEachFileIn(baseDir: string, namePrefix: st
 
 		const archiveName = `${namePrefix}-${filenameWithoutExtension}`
 
-		await createNamedTarball(filePath, archiveName, tarballDir)
+		await createNamedTarball(filePath, archiveName, tarballDir, callbacks)
 	}
 }

package/src/utilities/Utilities.ts

@@ -8,7 +8,8 @@ import { ChildProcessWithoutNullStreams } from 'node:child_process'
 import { inspect } from 'node:util'
 import { TypedArray, TypedArrayConstructor } from '../typings/TypedArray.js'
 import { encodeHex } from '../encodings/Hex.js'
-import { Timer } from './Timer.js'
+import { OperationCallbacks } from '../api/Common.js'
+import { parseJSON5, stringifyJSON5 } from 'quick-json5'
 
 const log = logToStderr
 
@@ -125,20 +126,8 @@ export function roundToDigits(val: number, digits = 3) {
 }
 
 export function sleep(timeMs: number) {
-	const timer = new Timer()
-
 	return new Promise<void>((resolve) => {
-		const tickCallback = () => {
-			if (timer.elapsedTime < timeMs) {
-				//setImmediate(tickCallback)
-				setTimeout(tickCallback, 0)
-			} else {
-				resolve()
-			}
-		}
-
-		//setImmediate(tickCallback)
-		setTimeout(tickCallback, 0)
+		setTimeout(resolve, timeMs)
 	})
 }
 
@@ -156,23 +145,19 @@ export function printMatrix(matrix: Float32Array[]) {
 	}
 }
 
-export async function parseJson(jsonText: string, useJson5 = false) {
+export function parseJson(jsonText: string, useJson5 = false) {
 	if (useJson5) {
-		const JSON5 = await import('json5')
-
-		return JSON5.parse(jsonText)
+		return parseJSON5(jsonText)
 	} else {
 		return JSON.parse(jsonText)
 	}
 }
 
-export async function stringifyAndFormatJson(obj: any, useJson5 = false) {
+export function stringifyAndFormatJson(obj: any, useJson5 = false) {
 	let textContent: string
 
 	if (useJson5) {
-		const JSON5 = await import('json5')
-
-		textContent = JSON5.stringify(obj, undefined, 4)
+		textContent = stringifyJSON5(obj, undefined, 4)
 	} else {
 		textContent = JSON.stringify(obj, undefined, 4)
 	}
@@ -180,11 +165,11 @@ export async function stringifyAndFormatJson(obj: any, useJson5 = false) {
 	return textContent
 }
 
-export async function parseJSONAndGetType(str: string, useJson5 = false) {
+export function parseJSONAndGetType(str: string, useJson5 = false) {
 	let parsedValue: any = undefined
 
 	try {
-		parsedValue = await parseJson(str, useJson5)
+		parsedValue = parseJson(str, useJson5)
 	} catch (e) {
 	}
 
@@ -210,7 +195,6 @@ export async function parseJSONAndGetType(str: string, useJson5 = false) {
 	}
 }
 
-
 export function secondsToHMS(totalSeconds: number) {
 	let remainingSeconds = totalSeconds
 
@@ -428,12 +412,13 @@ export async function resolveModuleScriptPath(moduleName: string) {
 
 export async function runOperationWithRetries<R>(
 	operationFunc: () => Promise<R>,
-	logger: Logger,
 	operationName = 'Operation',
+	callbacks: OperationCallbacks,
 	delayBetweenRetries = 2000,
-	maxRetries = 200) {
+	maxRetries = 200,
+	errorFilter?: (e: any) => boolean) {
 
-	const { default: chalk } = await import('chalk')
+	const logger = new Logger(callbacks.logLevel)
 
 	for (let retryIndex = 1; retryIndex <= maxRetries; retryIndex++) {
 		try {
@@ -441,25 +426,19 @@ export async function runOperationWithRetries<R>(
 
 			return result
 		} catch (e: any) {
-			const { shouldCancelCurrentTask } = await import('../server/Worker.js')
-
-			if (shouldCancelCurrentTask()) {
-				throw new Error('Canceled')
+			if (errorFilter && errorFilter(e) === false) {
+				throw e
 			}
 
-			logger.setAsActiveLogger()
-
-			logger.logTitledMessage(`Error`, e.message, chalk.redBright, 'error')
+			logger.logTitledMessage(`Error`, e.message, 'error')
 			logger.log('', 'error')
-			logger.logTitledMessage(`${operationName} failed`, `Trying again in ${delayBetweenRetries}ms..`, chalk.redBright, 'error')
+			logger.logTitledMessage(`${operationName} failed`, `Trying again in ${delayBetweenRetries}ms..`, 'error')
 
 			await sleep(delayBetweenRetries)
 
 			logger.log(``, 'warning')
-			logger.logTitledMessage(`Starting retry attempt`, `${retryIndex} / ${maxRetries}`, chalk.yellowBright, 'warning')
+			logger.logTitledMessage(`Starting retry attempt`, `${retryIndex} / ${maxRetries}`, 'warning')
 			logger.log(``, 'warning')
-
-			logger.unsetAsActiveLogger()
 		}
 	}
 
@@ -588,3 +567,43 @@ export function getTopKIndexes(values: ArrayLike<number>, topCount: number, sort
 
 	return topKIndexes
 }
+
+export function isValidHttpUrl(urlString: string): boolean {
+	if (typeof urlString !== 'string' || urlString.trim() === '') {
+		return false
+	}
+
+	try {
+		const url = new URL(urlString)
+
+		const allowedProtocols = ['http:', 'https:']
+
+		if (!allowedProtocols.includes(url.protocol)) {
+			return false;
+		}
+
+		if (!url.hostname) {
+			return false
+		}
+
+		return true
+	} catch (error) {
+		return false
+	}
+}
+
+export function throwIfAborted(abortSignal?: AbortSignal, cleanupFunction?: Function) {
+	if (!abortSignal) {
+		return
+	}
+
+	try {
+		abortSignal.throwIfAborted()
+	} catch (error) {
+		if (cleanupFunction) {
+			cleanupFunction()
+		}
+
+		throw error
+	}
+}

package/src/utilities/WebReader.ts

@@ -1,30 +1,35 @@
 import { Readability } from '@mozilla/readability'
 import { JSDOM, VirtualConsole } from 'jsdom'
-import { request } from 'gaxios'
 import { Logger } from './Logger.js'
 import { getChromeOnWindowsHeaders } from './BrowserRequestHeaders.js'
 import { convertHtmlToText } from './StringUtilities.js'
+import { requestHttp } from 'easier-http-request'
+import { OperationCallbacks } from '../api/Common.js'
+
+export async function fetchDocumentText(url: string, callbacks: OperationCallbacks) {
+	const progressLogger = new Logger(callbacks.logLevel)
 
-export async function fetchDocumentText(url: string) {
-	const progressLogger = new Logger()
 	progressLogger.start(`Fetch ${url}`)
 
 	const parsedUrl = new URL(url)
 	const origin = parsedUrl.origin
 
-	const response = await request<string>({
+	const response = await requestHttp({
 		url,
-		responseType: 'text',
 
 		headers: getChromeOnWindowsHeaders({
 			origin: origin,
 			referrer: `${origin}/`
 		}),
+
+		abortSignal: callbacks?.abortSignal
 	})
 
+	const responseBodyText = await response.text()
+
 	progressLogger.start(`Parse document body`)
 
-	const doc = new JSDOM(response.data, {
+	const doc = new JSDOM(responseBodyText, {
 		url,
 		virtualConsole: new VirtualConsole()
 	})

package/src/utilities/WikipediaReader.ts

@@ -1,11 +1,7 @@
+import { OperationCallbacks } from '../api/Common.js'
 import { splitToParagraphs, wordCharacterRegExp } from '../nlp/Segmentation.js'
-import { Logger } from './Logger.js'
-
-export async function parseWikipediaArticle(articleName: string, language: string) {
-	const logger = new Logger()
-
-	await logger.startAsync('Fetching Wikipedia article')
 
+export async function fetchAndParseWikipediaArticle(articleName: string, language: string, callbacks: OperationCallbacks) {
 	const { default: wtf } = await import('wtf_wikipedia')
 
 	const document = await wtf.fetch(articleName, language)
@@ -35,7 +31,5 @@ export async function parseWikipediaArticle(articleName: string, language: strin
 		}
 	}
 
-	logger.end()
-
 	return sectionsText
 }

package/src/voice-activity-detection/AdaptiveGateVAD.ts

@@ -1,14 +1,19 @@
-import { RawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js';
-import { BiquadFilter, createHighpassFilter, createLowpassFilter } from '../dsp/BiquadFilter.js';
-import { DecayingPeakEstimator } from '../dsp/DecayingPeakEstimator.js';
-import { LoudnessEstimator } from '../dsp/LoudnessEstimator.js';
-import { extendDeep } from '../utilities/ObjectUtilities.js';
-import { Timeline } from '../utilities/Timeline.js';
-import { logToStderr } from '../utilities/Utilities.js';
+import { VoiceActivityDetectionCallbacks } from '../api/VoiceActivityDetection.js'
+import { RawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
+import { BiquadFilter, createHighpassFilter, createLowpassFilter } from '../dsp/BiquadFilter.js'
+import { DecayingPeakEstimator } from '../dsp/DecayingPeakEstimator.js'
+import { LoudnessEstimator } from '../dsp/LoudnessEstimator.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { Timeline } from '../utilities/Timeline.js'
+import { logToStderr } from '../utilities/Utilities.js'
 
 const log = logToStderr
 
-export async function detectVoiceActivity(rawAudio: RawAudio, options: AdaptiveGateVADOptions) {
+export async function detectVoiceActivity(
+	rawAudio: RawAudio,
+	options: AdaptiveGateVADOptions,
+	callbacks: VoiceActivityDetectionCallbacks) {
+
 	const channelCount = rawAudio.audioChannels.length
 	const sampleCount = rawAudio.audioChannels[0].length
 	const sampleRate = rawAudio.sampleRate

package/src/voice-activity-detection/WebRtcVAD.ts

@@ -1,8 +1,6 @@
 import * as AudioBufferConversion from '../audio/AudioBufferConversion.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
 
-import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
-
 export async function detectVoiceActivity(rawAudio: RawAudio, frameDuration: 10 | 20 | 30 = 10, mode: 0 | 1 | 2 | 3 = 0) {
 	if (rawAudio.sampleRate != 16000) {
 		throw new Error('Audio sample rate must be 16KHz')
@@ -16,6 +14,7 @@ export async function detectVoiceActivity(rawAudio: RawAudio, frameDuration: 10
 export async function fvad(samples: Int16Array, sampleRate: number, frameDuration: 10 | 20 | 30, mode: 0 | 1 | 2 | 3) {
 	const m = await getFvadInstance()
 
+	const { wrapEmscriptenModuleHeap } = await import('wasm-heap-manager')
 	const wasmHeap = wrapEmscriptenModuleHeap(m)
 
 	const fvad_new = m._fvad_new