echogarden 0.11.12 → 0.11.13

package/src/tests/Test.ts

@@ -0,0 +1,24 @@
+import { getRepetitionScoreRelativeToFirstSubstring, logToStderr, setupProgramTerminationListeners } from "../utilities/Utilities.js"
+import { makeTarballsForInstalledPackages } from "../utilities/TarballMaker.js"
+import { testEspeakSynthesisWithPrePhonemizedInputs, testKirshenbaumPhonemization } from "../synthesis/EspeakTTS.js"
+
+const log = logToStderr
+
+setupProgramTerminationListeners()
+//process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
+//process.env.http_proxy = "http://localhost:8080"
+
+//const testText = `After a while, finding that nothing more happened, she decided on going into the garden at once; but, alas for poor Alice! when she got to the door, she found she had forgotten the little golden key, and when she went back to the table for it, she found she could not possibly reach it: she could see it quite plainly through the glass, and she tried her best to climb up one of the legs of the table, but it was too slippery; and when she had tired herself out with trying, the poor little thing sat down and cried.`
+
+//await testKirshenbaumPhonemization(testText)
+
+//await makeTarballsForInstalledPackages(true)
+
+//getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'c', 'a', 'b', 'c'])
+//getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'd', 'a', 'b', 'a', 'd'])
+//getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'b', 'c', 'a', 'b', 'a', 'b'])
+//getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'a', 'a', 'a', 'b'])
+//getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'c', 'a', 'b', 'a', 'c', 'a'])
+//getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'b', 'a', 'a', 'a', 'b'])
+
+process.exit(0)

package/src/text-language-detection/FastTextLanguageDetection.ts

@@ -0,0 +1,53 @@
+import path from 'path'
+import type { LanguageDetectionResults } from '../api/LanguageDetection.js'
+import { languageCodeToName } from '../utilities/Locale.js'
+import { OpenPromise } from '../utilities/OpenPromise.js'
+import { resolveModuleMainPath, roundToDigits } from '../utilities/Utilities.js'
+
+let fastTextLanguageDetectionModel: any
+
+export async function detectLanguage(text: string) {
+	const model = await loadLanguageDetectionModel()
+
+	const predictionsVector = model.predict(text, -1, 0.0)
+
+	const predictions: LanguageDetectionResults = []
+
+	for (let i = 0; i < predictionsVector.size(); i++) {
+		const prediction = predictionsVector.get(i)
+		const languageCode = prediction[1].substring(9)
+		const probability = roundToDigits(prediction[0], 3)
+
+		predictions.push({
+			language: languageCode,
+			languageName: languageCodeToName(languageCode),
+			probability
+		})
+	}
+
+	return predictions
+}
+
+async function loadLanguageDetectionModel() {
+	if (fastTextLanguageDetectionModel) {
+		return fastTextLanguageDetectionModel
+	}
+
+	const { FastText, addOnPostRun } = await import('@echogarden/fasttext-wasm')
+
+	const resultPromise = new OpenPromise<any>()
+
+	addOnPostRun(async () => {
+		const fastText = new FastText()
+
+		const moduleMainPath = await resolveModuleMainPath('@echogarden/fasttext-wasm')
+		const modelPath = path.join(path.dirname(moduleMainPath), 'lid.176.ftz')
+
+		const model = await fastText.loadModel(modelPath)
+
+		fastTextLanguageDetectionModel = model
+		resultPromise.resolve(model)
+	})
+
+	return resultPromise.promise
+}

package/src/text-language-detection/TinyLDLanguageDetection.ts

@@ -0,0 +1,16 @@
+import type { LanguageDetectionResults } from '../api/LanguageDetection.js'
+import { detectAll } from 'tinyld'
+import { languageCodeToName } from '../utilities/Locale.js'
+
+export async function detectLanguage(text: string) {
+	const tinyldResults = detectAll(text)
+
+	const results: LanguageDetectionResults =
+		tinyldResults.map(result => ({
+			language: result.lang,
+			languageName: languageCodeToName(result.lang),
+			probability: result.accuracy
+		}))
+
+	return results
+}

package/src/typings/Fillers.d.ts

@@ -0,0 +1,41 @@
+declare module 'moving-median'
+declare module 'html-to-text'
+declare module 'cldr-segmentation'
+
+declare module 'html-escaper' {
+	export function escape(str: string): string
+	export function unescape(str: string): string
+}
+
+declare module 'sam-js'
+declare module 'winax'
+
+declare module 'tinyld' {
+	export declare function detect(text: string, opts?: any): string
+
+	export declare function detectAll(text: string, opts?: any): {
+		lang: string
+		accuracy: number
+	}[]
+}
+
+declare module 'command-exists' {
+	function commandExists(commandName: string): Promise<string>
+
+	export default commandExists
+}
+
+declare module 'wtf_wikipedia'
+
+declare module 'kuromoji'
+
+declare module '@echogarden/espeak-ng-emscripten'
+declare module '@echogarden/svoxpico-wasm'
+declare module '@echogarden/fasttext-wasm'
+declare module '@echogarden/rubberband-wasm'
+declare module '@echogarden/rnnoise-wasm'
+declare module '@echogarden/fvad-wasm'
+declare module '@echogarden/sonic-wasm'
+declare module '@echogarden/kissfft-wasm'
+declare module '@echogarden/speex-resampler-wasm'
+declare module '@echogarden/vosk'

package/src/utilities/BinaryArrayConversion.ts

@@ -0,0 +1,159 @@
+// Typed arrays to Buffer (little endian) conversions
+
+// int8 <-> bufferLE
+export function int8ToBuffer(ints: Int8Array) {
+	const buffer = Buffer.alloc(ints.length)
+
+	for (let i = 0; i < ints.length; i++) {
+		buffer[i] = ints[i] + 128
+	}
+
+	return buffer
+}
+
+export function bufferToInt8(buffer: Buffer) {
+	const result = new Int8Array(buffer.length)
+
+	for (let i = 0; i < result.length; i++) {
+		result[i] = buffer[i] - 128
+	}
+
+	return result
+}
+
+// int16 <-> bufferLE
+export function int16ToBufferLE(ints: Int16Array) {
+	const buffer = Buffer.alloc(ints.length * 2)
+
+	for (let i = 0; i < ints.length; i++) {
+		buffer.writeInt16LE(ints[i], i * 2)
+	}
+
+	return buffer
+}
+
+export function bufferLEToInt16(buffer: Buffer) {
+	const result = new Int16Array(buffer.length / 2)
+
+	for (let i = 0; i < result.length; i++) {
+		result[i] = buffer.readInt16LE(i * 2)
+	}
+
+	return result
+}
+
+// int24 <-> bufferLE (uses int32 for storage)
+export function int24ToBufferLE(ints: Int32Array) {
+	const buffer = Buffer.alloc(ints.length * 3)
+
+	for (let i = 0; i < ints.length; i++) {
+		const val = ints[i]
+		const encodedVal = val < 0 ? val + 0x1000000 : val
+
+		buffer[(i * 3) + 0] = (encodedVal >> 0) & 0xff
+		buffer[(i * 3) + 1] = (encodedVal >> 8) & 0xff
+		buffer[(i * 3) + 2] = (encodedVal >> 16) & 0xff
+	}
+
+	return buffer
+}
+
+export function bufferLEToInt24(buffer: Buffer) {
+	const result = new Int32Array(buffer.length / 3)
+
+	for (let i = 0; i < result.length; i++) {
+		const b0 = buffer[(i * 3) + 0]
+		const b1 = buffer[(i * 3) + 1]
+		const b2 = buffer[(i * 3) + 2]
+
+		const encodedVal = (b0 << 0) + (b1 << 8) + (b2 << 16)
+		result[i] = encodedVal > 0x800000 ? encodedVal - 0x1000000 : encodedVal
+	}
+
+	return result
+}
+
+// int32 <-> bufferLE
+export function int32ToBufferLE(ints: Int32Array) {
+	const buffer = Buffer.alloc(ints.length * 4)
+
+	for (let i = 0; i < ints.length; i++) {
+		buffer.writeInt32LE(ints[i], i * 4)
+	}
+
+	return buffer
+}
+
+export function bufferLEToInt32(buffer: Buffer) {
+	const result = new Int32Array(buffer.length / 4)
+
+	for (let i = 0; i < result.length; i++) {
+		result[i] = buffer.readInt32LE(i * 4)
+	}
+
+	return result
+}
+
+// float32 <-> bufferLE
+export function float32ToBufferLE(floats: Float32Array) {
+	const buffer = Buffer.alloc(floats.length * 4)
+
+	for (let i = 0; i < floats.length; i++) {
+		buffer.writeFloatLE(floats[i], i * 4)
+	}
+
+	return buffer
+}
+
+export function bufferLEToFloat32(buffer: Buffer) {
+	const result = new Float32Array(buffer.length / 4)
+
+	for (let i = 0; i < result.length; i++) {
+		result[i] = buffer.readFloatLE(i * 4)
+	}
+
+	return result
+}
+
+// float64 <-> bufferLE
+export function float64ToBufferLE(floats: Float64Array) {
+	const buffer = Buffer.alloc(floats.length * 8)
+
+	for (let i = 0; i < floats.length; i++) {
+		buffer.writeDoubleLE(floats[i], i * 8)
+	}
+
+	return buffer
+}
+
+export function bufferLEToFloat64(buffer: Buffer) {
+	const result = new Float64Array(buffer.length / 8)
+
+	for (let i = 0; i < result.length; i++) {
+		result[i] = buffer.readDoubleLE(i * 8)
+	}
+
+	return result
+}
+
+// float64 <-> float32
+export function float64Tofloat32(doubles: Float64Array) {
+	const floats = new Float32Array(doubles.length)
+
+	for (let i = 0; i < doubles.length; i++) {
+		floats[i] = doubles[i]
+	}
+
+	return floats
+}
+
+export function float32Tofloat64(floats: Float32Array) {
+	const doubles = new Float64Array(floats.length)
+
+	for (let i = 0; i < floats.length; i++) {
+		doubles[i] = floats[i]
+	}
+
+	return doubles
+}
+

package/src/utilities/Compression.ts

@@ -0,0 +1,91 @@
+import path from 'node:path'
+import { Logger } from './Logger.js'
+import { readdir, stat } from './FileSystem.js'
+
+export async function createTarball(filePath: string, outputFile: string, prefixPath = "") {
+	const pathStat = await stat(filePath)
+
+	if (pathStat.isDirectory()) {
+		await createTarballForDir(filePath, outputFile, prefixPath)
+	} else {
+		await createTarballForFile(filePath, outputFile, prefixPath)
+	}
+}
+
+export async function createTarballForFile(filePath: string, outputFile: string, prefixPath = "") {
+	const logger = new Logger()
+
+	logger.start(`Creating ${prefixPath || path.basename(outputFile)}`)
+
+	const { default: tar } = await import('tar')
+
+	const inputFileStat = await stat(filePath)
+
+	if (!inputFileStat.isFile()) {
+		throw new Error(`${filePath} is not a file`)
+	}
+
+	const filename = path.basename(filePath)
+	const dirname = path.dirname(filePath)
+
+	await tar.create({
+		gzip: { level: 9, memLevel: 9 },
+		file: outputFile,
+		cwd: dirname,
+		prefix: prefixPath,
+		mode: 0o775,
+
+		filter: (path, stat) => {
+			stat.mode |= 0o775
+
+			return true
+		}
+
+	}, [filename])
+
+	logger.end()
+}
+
+export async function createTarballForDir(inputDir: string, outputFile: string, prefixPath = "") {
+	const logger = new Logger()
+
+	logger.start(`Creating ${prefixPath || path.basename(outputFile)}`)
+
+	const { default: tar } = await import('tar')
+
+	const inputDirStat = await stat(inputDir)
+
+	if (!inputDirStat.isDirectory()) {
+		throw new Error(`${inputDir} is not a directory`)
+	}
+
+	const filesInBaseDir = await readdir(inputDir)
+
+	await tar.create({
+		gzip: { level: 9, memLevel: 9 },
+		file: outputFile,
+		cwd: inputDir,
+		prefix: prefixPath,
+		mode: 0o775,
+
+		filter: (path, stat) => {
+			stat.mode |= 0o775
+
+			return true
+		}
+
+	}, filesInBaseDir)
+
+	logger.end()
+}
+
+export async function extractTarball(filepath: string, outputDir: string) {
+	const { default: tar } = await import('tar')
+
+	await tar.extract({
+		file: filepath,
+		cwd: outputDir,
+		preserveOwner: false,
+		//noChmod: true
+	})
+}

package/src/utilities/FileDownloader.ts

@@ -0,0 +1,201 @@
+import { GaxiosOptions, request } from "gaxios"
+import { Readable } from "stream"
+import { getRandomHexString, writeToStderr } from "./Utilities.js"
+import { OpenPromise } from "./OpenPromise.js"
+
+import { Timer } from "./Timer.js"
+import { Logger } from "./Logger.js"
+import path from "node:path"
+import { extractTarball } from "./Compression.js"
+import { createWriteStream, move, remove, readdir, ensureDir } from "./FileSystem.js"
+import chalk from "chalk"
+
+export async function downloadAndExtractTarball(options: GaxiosOptions, targetDir: string, baseTempPath: string, displayName = "archive") {
+	const logger = new Logger()
+
+	const randomID = getRandomHexString(16).toLowerCase()
+	const tempTarballPath = path.join(baseTempPath, `/${randomID}.tarball`)
+	const tempDirPath = path.join(baseTempPath, `/${randomID}`)
+	await ensureDir(tempDirPath)
+
+	await downloadFile(options, tempTarballPath, `${chalk.cyanBright('Downloading')} ${chalk.greenBright(displayName)}`)
+	logger.end()
+
+	logger.start(`Extracting ${displayName}`)
+
+	await extractTarball(tempTarballPath, tempDirPath)
+
+	await remove(tempTarballPath)
+
+	for (const filename of await readdir(tempDirPath)) {
+		const sourceFilePath = path.join(tempDirPath, filename)
+		const targetFilePath = path.join(targetDir, filename)
+
+		await move(sourceFilePath, targetFilePath)
+	}
+
+	await remove(tempDirPath)
+
+	logger.end()
+}
+
+export async function downloadFile(options: GaxiosOptions, targetFilePath: string, prompt = "Downloading") {
+	const write = writeToStderr
+
+	const downloadPromise = new OpenPromise<void>()
+
+	const timer = new Timer()
+
+	options.responseType = 'stream'
+
+	const response = await request<Readable>(options)
+
+	const ttyOutput = process.stderr.isTTY === true
+
+	write(`\n${prompt}.. `)
+
+	const rateAveragingWindowSeconds = 5.0
+
+	let downloadStarted = false
+	let downloadedBytes = 0
+	let totalBytes: number | undefined = undefined
+
+	const statusUpdateInterval = 250
+
+	let lastString = prompt
+
+	const downloadStateHistory: { time: number, downloadedMBytes: number }[] = []
+
+	function updateStatus() {
+		if (!downloadStarted) {
+			return
+		}
+
+		const totalMBytes = (totalBytes || 0) / 1000 / 1000
+		const downloadedMBytes = downloadedBytes / 1000 / 1000
+
+		const elapsedTime = timer.elapsedTimeSeconds
+		const cumulativeDownloadRate = downloadedMBytes / elapsedTime
+
+		const windowStartRecord = downloadStateHistory.find(r => r.time >= timer.elapsedTimeSeconds - rateAveragingWindowSeconds)
+
+		let windowDownloadRate: number
+
+		if (windowStartRecord) {
+			windowDownloadRate = (downloadedMBytes - windowStartRecord.downloadedMBytes) / (elapsedTime - windowStartRecord.time)
+		} else {
+			windowDownloadRate = cumulativeDownloadRate
+		}
+
+		downloadStateHistory.push({ time: elapsedTime, downloadedMBytes })
+
+		const isLastUpdate = downloadedMBytes == totalMBytes
+
+		const downloadedMbytesStr = downloadedMBytes.toFixed(2)
+		const totalMbytesStr = totalMBytes.toFixed(2)
+		const downloadRateStr = windowDownloadRate.toFixed(2)
+		const cumulativeDownloadRateStr = cumulativeDownloadRate.toFixed(2)
+
+		if (ttyOutput) {
+			let newString: string
+
+			if (totalBytes != undefined) {
+				const percentage = (downloadedMBytes / totalMBytes) * 100
+
+				newString = `${prompt}.. ${downloadedMbytesStr}MB/${totalMbytesStr}MB (${chalk.blueBright(percentage.toFixed(1) + '%')}, ${timer.elapsedTimeSeconds.toFixed(1)}s, ${downloadRateStr}MB/s)`
+			} else {
+				newString = `${prompt}.. ${downloadedMbytesStr}MB (${timer.elapsedTimeSeconds.toFixed(1)}s, ${downloadRateStr}MB/s)`
+			}
+
+			if (newString != lastString) {
+				write('\r')
+				write(newString)
+			}
+
+			lastString = newString
+		} else {
+			if (totalBytes == undefined) {
+				return
+			}
+
+			const percent = downloadedBytes / totalBytes
+			const percentDisplay = `${(Math.floor(percent * 10) * 10).toString()}%`
+
+			if (lastString == prompt) {
+				write(`(${totalMbytesStr}MB): `)
+			}
+
+			if (percentDisplay != lastString) {
+				write(percentDisplay)
+
+				if (percent == 1.0) {
+					write(` (${timer.elapsedTimeSeconds.toFixed(2)}s, ${cumulativeDownloadRateStr}MB/s)`)
+				} else {
+					write(` `)
+				}
+
+				lastString = percentDisplay
+			}
+		}
+	}
+
+	const partialFilePath = `${targetFilePath}.${getRandomHexString(16)}.partial`
+	const fileWriteStream = createWriteStream(partialFilePath, { encoding: "binary", autoClose: true })
+
+	let statusInterval = setInterval(() => {
+		updateStatus()
+	}, statusUpdateInterval)
+
+	response.data.on("data", (chunk: Uint8Array) => {
+		try {
+			const contentLengthString = response.headers['content-length']
+
+			totalBytes = contentLengthString != undefined ? parseInt(contentLengthString) : undefined
+
+			const chunkLength = chunk.length
+
+			fileWriteStream.write(chunk)
+
+			downloadedBytes += chunkLength
+
+			if (downloadStarted == false) {
+				downloadStarted = true
+			}
+		} catch (err) {
+			clearInterval(statusInterval)
+
+			downloadPromise.reject(err)
+		}
+	})
+
+	response.data.on("end", async () => {
+		try {
+			clearInterval(statusInterval)
+			updateStatus()
+
+			fileWriteStream.end()
+
+			write("\n")
+
+			await move(partialFilePath, targetFilePath)
+
+			downloadPromise.resolve()
+		} catch (err) {
+			clearInterval(statusInterval)
+			downloadPromise.reject(err)
+		}
+	})
+
+	response.data.on("error", async (err: any) => {
+		try {
+			clearInterval(statusInterval)
+
+			fileWriteStream.end()
+			await remove(partialFilePath)
+		} finally {
+			downloadPromise.reject(err)
+		}
+	})
+
+	return downloadPromise.promise
+}