echogarden 1.8.7 → 2.0.0

package/src/utilities/FileSystem.ts

@@ -2,157 +2,168 @@ import * as fsExtra from 'fs-extra/esm'
 import gracefulFS from 'graceful-fs'
 import * as os from 'node:os'
 
-import path from 'node:path'
 import { promisify } from 'node:util'
-import { OpenPromise } from './OpenPromise.js'
-import { getRandomHexString } from './Utilities.js'
+import { getRandomHexString, parseJson, stringifyAndFormatJson } from './Utilities.js'
 import { appName } from '../api/Common.js'
-import { getAppTempDir } from './PathUtilities.js'
-import { writeFile as nodeFsWriteFile } from 'node:fs/promises'
+import { getAppTempDir, getDirName, joinPath, normalizePath, parsePath } from './PathUtilities.js'
 
-export const readFile = promisify(gracefulFS.readFile)
-//export const gracefulFsWriteFile = promisify(gracefulFS.writeFile)
+import { createDynamicUint8Array } from '../data-structures/DynamicTypedArray.js'
+import { ChunkedUtf8Decoder } from '../encodings/Utf8.js'
+import { FileWriter } from './FileWriter.js'
+import { FileReader } from './FileReader.js'
 
-export const readdir = promisify(gracefulFS.readdir)
-export const stat = promisify(gracefulFS.stat)
 export const open = promisify(gracefulFS.open)
 export const close = promisify(gracefulFS.close)
+
+export const read = promisify(gracefulFS.read)
+export const write = promisify(gracefulFS.write)
+
+export const existsSync = gracefulFS.existsSync
+
+export const stat = promisify(gracefulFS.stat)
 export const chmod = promisify(gracefulFS.chmod)
-export const copyFile = promisify(gracefulFS.copyFile)
 export const access = promisify(gracefulFS.access)
 
-export const createReadStream = gracefulFS.createReadStream
-export const createWriteStream = gracefulFS.createWriteStream
-export const existsSync = gracefulFS.existsSync
+export const readdir = promisify(gracefulFS.readdir)
+export const copyFile = promisify(gracefulFS.copyFile)
 
 export const remove = fsExtra.remove
 export const copy = fsExtra.copy
-// const outputFile = fsExtra.outputFile
 
-export async function readDirRecursive(dir: string, fileFilter?: (filePath: string) => boolean) {
-	if (!(await stat(dir)).isDirectory()) {
-		throw new Error(`'${dir}' is not a directory`)
-	}
+///////////////////////////////////////////////////////////////////////////////////////////
+// File read operations
+///////////////////////////////////////////////////////////////////////////////////////////
+export async function readFileAsBinary(filePath: string) {
+	const chunkSize = 2 ** 20
 
-	const filenamesInDir = await readdir(dir)
-	const filesInDir = filenamesInDir.map(filename => path.join(dir, filename))
+	const fileInfo = await stat(filePath)
+	const fileSize = fileInfo.size
 
-	const result: string[] = []
-	const subDirectories: string[] = []
+	const fileReader = new FileReader(filePath)
+	const buffer = new Uint8Array(chunkSize)
+	const result = createDynamicUint8Array(fileSize)
 
-	for (const filePath of filesInDir) {
-		if ((await stat(filePath)).isDirectory()) {
-			subDirectories.push(filePath)
-		} else {
-			if (fileFilter && !fileFilter(filePath)) {
-				continue
-			}
+	while (!fileReader.isFinished) {
+		const chunk = await fileReader.readChunk(buffer)
 
-			result.push(filePath)
-		}
+		result.addArray(chunk)
 	}
 
-	for (const subDirectory of subDirectories) {
-		const filesInSubdirectory = await readDirRecursive(subDirectory, fileFilter)
-		result.push(...filesInSubdirectory)
-	}
-
-	return result
+	return result.toTypedArray()
 }
 
-export async function isFileIsUpToDate(filePath: string, timeRangeSeconds: number) {
-	const fileUpdateTime = (await stat(filePath)).mtime.valueOf()
+export async function readFileAsUtf8(filePath: string) {
+	const chunkSize = 2 ** 20
 
-	const currentTime = (new Date()).valueOf()
+	const fileReader = new FileReader(filePath)
+	const buffer = new Uint8Array(chunkSize)
 
-	const differenceInMilliseconds = currentTime - fileUpdateTime
+	const result = new ChunkedUtf8Decoder()
 
-	const differenceInSeconds = differenceInMilliseconds / 1000
+	while (!fileReader.isFinished) {
+		const chunk = await fileReader.readChunk(buffer)
+
+		result.writeChunk(chunk)
+	}
 
-	return differenceInSeconds <= timeRangeSeconds
+	return result.toString()
 }
 
-export async function computeFileSha256Hex(filePath: string) {
-	const resultOpenPromise = new OpenPromise<string>()
+export async function readAndParseJsonFile(jsonFilePath: string, useJson5 = false) {
+	const textContent = await readFileAsUtf8(jsonFilePath)
 
-	const crypto = await import('crypto')
-	const hash = crypto.createHash('sha256')
+	return parseJson(textContent, useJson5)
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// File write operations
+///////////////////////////////////////////////////////////////////////////////////////////
+export async function writeFile(filePath: string, content: Uint8Array | string) {
+	if (content instanceof Uint8Array) {
+		return writeBinaryFile(filePath, content)
+	} else if (typeof content === 'string') {
+		return writeUtf8File(filePath, content)
+	} else {
+		throw new Error(`Content can only be a Uint8Array or string`)
+	}
+}
 
-	const readStream = createReadStream(filePath)
+export async function writeBinaryFile(filePath: string, content: Uint8Array) {
+	const maxChunkSize = 2 ** 20
 
-	readStream.on('data', data => hash.update(data))
-	readStream.on('error', error => resultOpenPromise.reject(error))
-	readStream.on('end', () => resultOpenPromise.resolve(hash.digest('hex')))
+	const fileDir = getDirName(filePath)
 
-	return resultOpenPromise.promise
-}
+	await ensureDir(fileDir)
 
-export async function readAndParseJsonFile(jsonFilePath: string, useJson5 = false) {
-	const fileContent = await readFile(jsonFilePath, { encoding: 'utf-8' })
+	const fileWriter = new FileWriter(filePath)
 
-	if (useJson5) {
-		const { default: JSON5 } = await import('json5')
+	let readOffset = 0
 
-		return JSON5.parse(fileContent)
-	} else {
-		return JSON.parse(fileContent)
+	while (true) {
+		const chunk = content.subarray(readOffset, readOffset + maxChunkSize)
+
+		if (chunk.length === 0) {
+			break
+		}
+
+		await fileWriter.write(chunk)
+
+		readOffset += chunk.length
 	}
+
+	await fileWriter.dispose()
 }
 
-export async function writeFile(filePath: string, data: string | NodeJS.ArrayBufferView, options?: fsExtra.WriteFileOptions) {
-	const fileDir = path.dirname(filePath)
+export async function writeUtf8File(filePath: string, content: string) {
+	const maxChunkSize = 2 ** 20
+
+	const fileDir = getDirName(filePath)
 
 	await ensureDir(fileDir)
 
-	return nodeFsWriteFile(filePath, data, options)
-}
+	const fileWriter = new FileWriter(filePath)
 
-export async function writeFileSafe(filePath: string, data: string | NodeJS.ArrayBufferView, options?: fsExtra.WriteFileOptions) {
-	const tempDir = getAppTempDir(appName)
-	const tempFilePath = path.join(tempDir, `${getRandomHexString(16)}.partial`)
+	const textEncoder = new TextEncoder()
 
-	await writeFile(tempFilePath, data, options)
+	let readOffset = 0
 
-	await move(tempFilePath, filePath)
-}
+	while (true) {
+		const stringChunk = content.substring(readOffset, readOffset + maxChunkSize)
 
-export function getAppDataDir(appName: string) {
-	let dataDir: string
+		if (stringChunk.length === 0) {
+			break
+		}
 
-	const platform = process.platform
-	const homeDir = os.homedir()
+		const chunk = textEncoder.encode(stringChunk)
 
-	if (platform == 'win32') {
-		dataDir = path.join(homeDir, 'AppData', 'Local', appName)
-	} else if (platform == 'darwin') {
-		dataDir = path.join(homeDir, 'Library', 'Application Support', appName)
-	} else if (platform == 'linux') {
-		dataDir = path.join(homeDir, '.local', 'share', appName)
-	} else {
-		throw new Error(`Unsupport platform ${platform}`)
+		await fileWriter.write(chunk)
+
+		readOffset += stringChunk.length
 	}
 
-	return dataDir
+	await fileWriter.dispose()
 }
 
-export async function chmodRecursive(rootPath: string, newMode: number) {
-	const rootPathStat = await stat(rootPath)
+export async function writeJsonFile(filePath: string, content: any, useJson5 = false) {
+	const textContent = await stringifyAndFormatJson(content, useJson5)
 
-	await chmod(rootPath, newMode)
+	await writeUtf8File(filePath, textContent)
+}
 
-	if (rootPathStat.isDirectory()) {
-		const fileList = await readdir(rootPath)
+export async function writeFileSafe(filePath: string, content: Uint8Array | string) {
+	const tempDir = getAppTempDir(appName)
+	const tempFilePath = joinPath(tempDir, `${getRandomHexString(16)}.partial`)
 
-		for (const filename of fileList) {
-			const filePath = path.join(rootPath, filename)
+	await writeFile(tempFilePath, content)
 
-			await chmodRecursive(filePath, newMode)
-		}
-	}
+	await move(tempFilePath, filePath)
 }
 
+///////////////////////////////////////////////////////////////////////////////////////////
+// Directory operations
+///////////////////////////////////////////////////////////////////////////////////////////
 export async function ensureDir(dirPath: string) {
-	dirPath = path.normalize(dirPath)
+	dirPath = normalizePath(dirPath)
 
 	if (existsSync(dirPath)) {
 		const dirStats = await stat(dirPath)
@@ -165,9 +176,75 @@ export async function ensureDir(dirPath: string) {
 	}
 }
 
+export async function testDirectoryIsWritable(dir: string) {
+	const testFileName = joinPath(dir, getRandomHexString(16))
+
+	try {
+		await fsExtra.createFile(testFileName)
+		await remove(testFileName)
+	} catch (e) {
+		return false
+	}
+
+	return true
+}
+
+export async function readDirRecursive(dir: string, pathFilter?: (filePath: string) => boolean) {
+	if (!(await stat(dir)).isDirectory()) {
+		throw new Error(`'${dir}' is not a directory`)
+	}
+
+	const filenamesInDir = await readdir(dir)
+	const filesInDir = filenamesInDir.map(filename => joinPath(dir, filename))
+
+	const result: string[] = []
+	const subDirectories: string[] = []
+
+	for (const filePath of filesInDir) {
+		if ((await stat(filePath)).isDirectory()) {
+			subDirectories.push(filePath)
+		} else {
+			if (pathFilter && !pathFilter(filePath)) {
+				continue
+			}
+
+			result.push(filePath)
+		}
+	}
+
+	for (const subDirectory of subDirectories) {
+		const filesInSubdirectory = await readDirRecursive(subDirectory, pathFilter)
+		result.push(...filesInSubdirectory)
+	}
+
+	return result
+}
+
+export function getAppDataDir(appName: string) {
+	let dataDir: string
+
+	const platform = process.platform
+	const homeDir = os.homedir()
+
+	if (platform == 'win32') {
+		dataDir = joinPath(homeDir, 'AppData', 'Local', appName)
+	} else if (platform == 'darwin') {
+		dataDir = joinPath(homeDir, 'Library', 'Application Support', appName)
+	} else if (platform == 'linux') {
+		dataDir = joinPath(homeDir, '.local', 'share', appName)
+	} else {
+		throw new Error(`Unsupport platform ${platform}`)
+	}
+
+	return dataDir
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// Copy operations
+///////////////////////////////////////////////////////////////////////////////////////////
 export async function move(source: string, dest: string) {
-	source = path.normalize(source)
-	dest = path.normalize(dest)
+	source = normalizePath(source)
+	dest = normalizePath(dest)
 
 	if (existsSync(dest)) {
 		const destPathExistsAndIsWritable = await existsAndIsWritable(dest)
@@ -176,7 +253,7 @@ export async function move(source: string, dest: string) {
 			throw new Error(`The destination path '${dest}' exists but is not writable. There may be a permissions or locking issue.`)
 		}
 	} else {
-		const destDir = path.parse(dest).dir
+		const destDir = parsePath(dest).dir
 		const destDirIsWritable = await testDirectoryIsWritable(destDir)
 
 		if (!destDirIsWritable) {
@@ -187,6 +264,9 @@ export async function move(source: string, dest: string) {
 	return fsExtra.move(source, dest, { overwrite: true })
 }
 
+///////////////////////////////////////////////////////////////////////////////////////////
+// Misc operations
+///////////////////////////////////////////////////////////////////////////////////////////
 export async function existsAndIsWritable(targetPath: string) {
 	try {
 		await access(targetPath, gracefulFS.constants.W_OK);
@@ -197,36 +277,49 @@ export async function existsAndIsWritable(targetPath: string) {
 	return true
 }
 
-export async function testDirectoryIsWritable(dir: string) {
-	const testFileName = path.join(dir, getRandomHexString(16))
+export async function chmodRecursive(rootPath: string, newMode: number) {
+	const rootPathStat = await stat(rootPath)
 
-	try {
-		await fsExtra.createFile(testFileName)
-		await remove(testFileName)
-	} catch (e) {
-		return false
+	await chmod(rootPath, newMode)
+
+	if (rootPathStat.isDirectory()) {
+		const fileList = await readdir(rootPath)
+
+		for (const filename of fileList) {
+			const filePath = joinPath(rootPath, filename)
+
+			await chmodRecursive(filePath, newMode)
+		}
 	}
+}
 
-	return true
+export async function isFileIsUpToDate(filePath: string, maxTimeDifferenceSeconds: number) {
+	const fileUpdateTime = (await stat(filePath)).mtime.valueOf()
+
+	const currentTime = (new Date()).valueOf()
+
+	const differenceInMilliseconds = currentTime - fileUpdateTime
+
+	const differenceInSeconds = differenceInMilliseconds / 1000
+
+	return differenceInSeconds <= maxTimeDifferenceSeconds
 }
 
-export async function copyFileAlternative(source: string, dest: string) {
-	return new Promise<void>((resolve, reject) => {
-		const readStream = createReadStream(source)
-		const writeStream = createWriteStream(dest)
+export async function computeFileSha256Hex(filePath: string) {
+	const crypto = await import('crypto')
+
+	const hash = crypto.createHash('sha256')
 
-		readStream.on('error', (err: any) => {
-			reject(err)
-		})
+	const fileReader = new FileReader(filePath)
+	const buffer = new Uint8Array(2 ** 16)
 
-		writeStream.on('error', (err: any) => {
-			reject(err)
-		})
+	while (!fileReader.isFinished) {
+		const chunk = await fileReader.readChunk(buffer)
 
-		readStream.pipe(writeStream)
+		hash.update(chunk)
+	}
 
-		readStream.on('end', () => {
-			resolve()
-		})
-	})
+	const result = hash.digest('hex')
+
+	return result
 }

package/src/utilities/FileWriter.ts

@@ -0,0 +1,72 @@
+import { write, open, close } from "./FileSystem.js"
+
+export class FileWriter {
+	private fileHandle?: number
+	private disposed = false
+	private writeOffset = 0
+
+	constructor(public readonly filePath: string) {
+	}
+
+	async write(chunk: Uint8Array) {
+		if (this.isDisposed) {
+			throw new Error(`FileWriter has been disposed`)
+		}
+
+		await this.openIfNeeded()
+
+		let chunkReadOffset = 0
+
+		while (chunkReadOffset < chunk.length) {
+			let bytesWritten: number
+
+			try {
+				({ bytesWritten } = await write(this.fileHandle!, chunk, chunkReadOffset, undefined, this.writeOffset))
+			} catch (e) {
+				await this.dispose()
+
+				throw e
+			}
+
+			chunkReadOffset += bytesWritten
+			this.writeOffset += bytesWritten
+		}
+	}
+
+	private async openIfNeeded() {
+		if (this.isDisposed) {
+			throw new Error(`FileWriter has been disposed`)
+		}
+
+		if (this.isOpened) {
+			return
+		}
+
+		this.fileHandle = await open(this.filePath, 'w')
+	}
+
+	async dispose() {
+		if (this.isDisposed) {
+			return
+		}
+
+		if (this.isOpened) {
+			try {
+				await close(this.fileHandle!)
+			} catch (e) {
+			}
+		}
+
+		this.disposed = true
+		this.writeOffset = 0
+		this.fileHandle = undefined
+	}
+
+	get isOpened() {
+		return this.fileHandle !== undefined
+	}
+
+	get isDisposed() {
+		return this.disposed
+	}
+}

package/src/utilities/NpmUtilities.ts

@@ -0,0 +1,23 @@
+import { request } from 'gaxios'
+
+export async function getPackageLatestVersion(packageName: string, timeout = 10000) {
+	const response = await request<any>({
+		method: 'GET',
+
+		url: `https://registry.npmjs.org/${packageName}/latest`,
+
+		params: {
+		},
+
+		headers: {
+		},
+
+		responseType: 'json',
+
+		timeout,
+	})
+
+	const result = response.data
+
+	return result.version
+}

package/src/utilities/PackageManager.ts

@@ -1,9 +1,8 @@
-import path from 'node:path'
 import { downloadAndExtractTarball } from './FileDownloader.js'
 import { getAppDataDir, ensureDir, existsSync, remove } from './FileSystem.js'
 import { appName } from '../api/Common.js'
 import { GaxiosOptions } from 'gaxios'
-import { getAppTempDir } from './PathUtilities.js'
+import { getAppTempDir, joinPath } from './PathUtilities.js'
 import { getGlobalOption } from '../api/GlobalOptions.js'
 
 export async function loadPackage(packageName: string) {
@@ -11,7 +10,7 @@ export async function loadPackage(packageName: string) {
 
 	const packagesPath = await ensureAndGetPackagesDir()
 
-	const packagePath = path.join(packagesPath, packageName)
+	const packagePath = joinPath(packagesPath, packageName)
 
 	if (existsSync(packagePath)) {
 		return packagePath
@@ -44,7 +43,7 @@ export async function removePackage(packageName: string) {
 
 	const packagesPath = await ensureAndGetPackagesDir()
 
-	const packagePath = path.join(packagesPath, packageName)
+	const packagePath = joinPath(packagesPath, packageName)
 
 	await remove(packagePath)
 }
@@ -52,7 +51,7 @@ export async function removePackage(packageName: string) {
 export async function ensureAndGetPackagesDir() {
 	const dataPath = getAppDataDir(appName)
 
-	const packagesPath = path.join(dataPath, 'packages')
+	const packagesPath = joinPath(dataPath, 'packages')
 
 	await ensureDir(packagesPath)
 
@@ -209,4 +208,8 @@ const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
 	// W2V-BERT models
 	'w2v-bert-2.0-int8': '20240517',
 	'w2v-bert-2.0-uint8': '20240517',
+
+	// NSNet2 models
+	'nsnet2-20ms-baseline': '20242610',
+	'nsnet2-20ms-48k-baseline': '20242610',
 }

package/src/utilities/PathUtilities.ts

@@ -1,14 +1,15 @@
-import path from 'node:path'
+import * as NodePath from 'node:path'
 import { fileURLToPath } from 'node:url'
 import * as os from 'node:os'
 
 export function getModuleRootDir() {
-	const currentScriptDir = path.dirname(fileURLToPath(import.meta.url))
-	return path.resolve(currentScriptDir, '..', '..')
+	const currentScriptDir = getDirName(fileURLToPath(import.meta.url))
+
+	return resolvePath(currentScriptDir, '..', '..')
 }
 
 export function resolveToModuleRootDir(relativePath: string) {
-	return path.resolve(getModuleRootDir(), relativePath)
+	return resolvePath(getModuleRootDir(), relativePath)
 }
 
 export function getLowercaseFileExtension(filename: string) {
@@ -28,14 +29,42 @@ export function getAppTempDir(appName: string) {
 	const homeDir = os.homedir()
 
 	if (platform == 'win32') {
-		tempDir = path.join(homeDir, 'AppData', 'Local', 'Temp', appName)
+		tempDir = joinPath(homeDir, 'AppData', 'Local', 'Temp', appName)
 	} else if (platform == 'darwin') {
-		tempDir = path.join(homeDir, 'Library', 'Caches', appName)
+		tempDir = joinPath(homeDir, 'Library', 'Caches', appName)
 	} else if (platform == 'linux') {
-		tempDir = path.join(homeDir, '.cache', appName)
+		tempDir = joinPath(homeDir, '.cache', appName)
 	} else {
 		throw new Error(`Unsupport platform ${platform}`)
 	}
 
 	return tempDir
 }
+
+export function joinPath(...paths: string[]) {
+	return NodePath.join(...paths)
+}
+
+export function resolvePath(...paths: string[]) {
+	return NodePath.resolve(...paths)
+}
+
+export function normalizePath(path: string) {
+	return NodePath.normalize(path)
+}
+
+export function getBaseName(path: string) {
+	return NodePath.basename(path)
+}
+
+export function getDirName(path: string) {
+	return NodePath.dirname(path)
+}
+
+export function getFileNameWithoutExtension(filePath: string) {
+	return NodePath.parse(filePath).name
+}
+
+export function parsePath(path: string) {
+	return NodePath.parse(path)
+}