echogarden 0.11.12 → 0.11.13

package/src/utilities/FileSystem.ts

@@ -0,0 +1,265 @@
+import * as fsExtra from 'fs-extra/esm'
+import gracefulFS from 'graceful-fs'
+import * as os from 'node:os'
+
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import { OpenPromise } from './OpenPromise.js'
+import { getRandomHexString, sha256AsHex } from './Utilities.js'
+import { appName } from '../api/Common.js'
+
+export const readFile = promisify(gracefulFS.readFile)
+//export const writeFile = promisify(gracefulFS.writeFile)
+export const readdir = promisify(gracefulFS.readdir)
+export const stat = promisify(gracefulFS.stat)
+export const open = promisify(gracefulFS.open)
+export const close = promisify(gracefulFS.close)
+export const chmod = promisify(gracefulFS.chmod)
+export const copyFile = promisify(gracefulFS.copyFile)
+export const access = promisify(gracefulFS.access)
+
+
+export const createReadStream = gracefulFS.createReadStream
+export const createWriteStream = gracefulFS.createWriteStream
+export const existsSync = gracefulFS.existsSync
+
+export const remove = fsExtra.remove
+export const copy = fsExtra.copy
+export const outputFile = fsExtra.outputFile
+
+export async function readDirRecursive(dir: string, fileFilter?: (filePath: string) => boolean) {
+	if (!(await stat(dir)).isDirectory()) {
+		throw new Error(`'${dir}' is not a directory`)
+	}
+
+	const filenamesInDir = await readdir(dir)
+	const filesInDir = filenamesInDir.map(filename => path.join(dir, filename))
+
+	const result: string[] = []
+	const subDirectories: string[] = []
+
+	for (const filePath of filesInDir) {
+		if ((await stat(filePath)).isDirectory()) {
+			subDirectories.push(filePath)
+		} else {
+			if (fileFilter && !fileFilter(filePath)) {
+				continue
+			}
+
+			result.push(filePath)
+		}
+	}
+
+	for (const subDirectory of subDirectories) {
+		const filesInSubdirectory = await readDirRecursive(subDirectory, fileFilter)
+		result.push(...filesInSubdirectory)
+	}
+
+	return result
+}
+
+export async function isFileIsUpToDate(filePath: string, timeRangeSeconds: number) {
+	const fileUpdateTime = (await stat(filePath)).mtime.valueOf()
+
+	const currentTime = (new Date()).valueOf()
+
+	const differenceInMilliseconds = currentTime - fileUpdateTime
+
+	const differenceInSeconds = differenceInMilliseconds / 1000
+
+	return differenceInSeconds <= timeRangeSeconds
+}
+
+export function getModuleRootDir() {
+	const currentScriptDir = path.dirname(fileURLToPath(import.meta.url))
+	return path.resolve(currentScriptDir, '..', '..')
+}
+
+export function resolveToModuleRootDir(relativePath: string) {
+	return path.resolve(getModuleRootDir(), relativePath)
+}
+
+export function getLowercaseFileExtension(filename: string) {
+	const fileExtensionIndex = filename.lastIndexOf(".")
+
+	if (fileExtensionIndex == -1) {
+		return ""
+	}
+
+	return filename.substring(fileExtensionIndex + 1).toLowerCase()
+}
+
+export async function computeFileSha256Hex(filePath: string) {
+	const resultOpenPromise = new OpenPromise<string>()
+
+	const crypto = await import('crypto')
+	const hash = crypto.createHash('sha256')
+
+	const readStream = createReadStream(filePath)
+
+	readStream.on('data', data => hash.update(data))
+	readStream.on('error', error => resultOpenPromise.reject(error))
+	readStream.on('end', () => resultOpenPromise.resolve(hash.digest('hex')))
+
+	return resultOpenPromise.promise
+}
+
+export async function readAndParseJsonFile(jsonFilePath: string, useJson5 = false) {
+	const fileContent = await readFile(jsonFilePath, { encoding: "utf-8" })
+
+	if (useJson5) {
+		const { default: JSON5 } = await import('json5')
+
+		return JSON5.parse(fileContent)
+	} else {
+		return JSON.parse(fileContent)
+	}
+}
+
+export async function writeFile(filePath: string, data: string | NodeJS.ArrayBufferView, options?: fsExtra.WriteFileOptions) {
+	return outputFile(filePath, data, options)
+}
+
+export async function writeFileSafe(filePath: string, data: string | NodeJS.ArrayBufferView, options?: fsExtra.WriteFileOptions) {
+	const tempDir = getAppTempDir(appName)
+	const tempFilePath = path.join(tempDir, `${getRandomHexString(16)}.partial`)
+
+	await writeFile(tempFilePath, data, options)
+
+	await move(tempFilePath, filePath)
+}
+
+export function getAppTempDir(appName: string) {
+	let tempDir: string
+
+	const platform = process.platform
+	const homeDir = os.homedir()
+
+	if (platform == "win32") {
+		tempDir = path.join(homeDir, "AppData", "Local", "Temp", appName)
+	} else if (platform == "darwin") {
+		tempDir = path.join(homeDir, "Library", "Caches", appName)
+	} else if (platform == "linux") {
+		tempDir = path.join(homeDir, ".cache", appName)
+	} else {
+		throw new Error(`Unsupport platform ${platform}`)
+	}
+
+	return tempDir
+}
+
+export function getAppDataDir(appName: string) {
+	let dataDir: string
+
+	const platform = process.platform
+	const homeDir = os.homedir()
+
+	if (platform == "win32") {
+		dataDir = path.join(homeDir, "AppData", "Local", appName)
+	} else if (platform == "darwin") {
+		dataDir = path.join(homeDir, "Library", "Application Support", appName)
+	} else if (platform == "linux") {
+		dataDir = path.join(homeDir, ".local", "share", appName)
+	} else {
+		throw new Error(`Unsupport platform ${platform}`)
+	}
+
+	return dataDir
+}
+
+export async function chmodRecursive(rootPath: string, newMode: number) {
+	const rootPathStat = await stat(rootPath)
+
+	await chmod(rootPath, newMode)
+
+	if (rootPathStat.isDirectory()) {
+		const fileList = await readdir(rootPath)
+
+		for (const filename of fileList) {
+			const filePath = path.join(rootPath, filename)
+
+			await chmodRecursive(filePath, newMode)
+		}
+	}
+}
+
+export async function ensureDir(dirPath: string) {
+	dirPath = path.normalize(dirPath)
+
+	if (existsSync(dirPath)) {
+		const dirStats = await stat(dirPath)
+
+		if (!dirStats.isDirectory()) {
+			throw new Error(`The path '${dirPath}' exists but is not a directory.`)
+		}
+	} else {
+		return fsExtra.ensureDir(dirPath)
+	}
+}
+
+export async function move(source: string, dest: string) {
+	source = path.normalize(source)
+	dest = path.normalize(dest)
+
+	if (existsSync(dest)) {
+		const destPathExistsAndIsWritable = await existsAndIsWritable(dest)
+
+		if (!destPathExistsAndIsWritable) {
+			throw new Error(`The destination path '${dest}' exists but is not writable. There may be a permissions or locking issue.`)
+		}
+	} else {
+		const destDir = path.parse(dest).dir
+		const destDirIsWritable = await testDirectoryIsWritable(destDir)
+
+		if (!destDirIsWritable) {
+			throw new Error(`The directory ${destDir} is not writable. There may be a permissions issue.`)
+		}
+	}
+
+	return fsExtra.move(source, dest, { overwrite: true })
+}
+
+export async function existsAndIsWritable(targetPath: string) {
+	try {
+		await access(targetPath, gracefulFS.constants.W_OK);
+	} catch {
+		return false
+	}
+
+	return true
+}
+
+export async function testDirectoryIsWritable(dir: string) {
+	const testFileName = path.join(dir, getRandomHexString(16))
+
+	try {
+		await fsExtra.createFile(testFileName)
+		await remove(testFileName)
+	} catch (e) {
+		return false
+	}
+
+	return true
+}
+
+export async function copyFileAlternative(source: string, dest: string) {
+	return new Promise<void>((resolve, reject) => {
+		const readStream = createReadStream(source)
+		const writeStream = createWriteStream(dest)
+
+		readStream.on('error', (err: any) => {
+			reject(err)
+		})
+
+		writeStream.on('error', (err: any) => {
+			reject(err)
+		})
+
+		readStream.pipe(writeStream)
+
+		readStream.on('end', () => {
+			resolve()
+		})
+	})
+}

package/src/utilities/Hashing.ts

@@ -0,0 +1,230 @@
+export function knuthMultiplicative(bytes: Buffer) {
+	let hash = 0
+
+	for (const byte of bytes) {
+		hash += Math.imul(byte, 2654435761)
+	}
+
+	return hash
+}
+
+export function xorShift32Hash(bytes: Buffer) {
+	let s = 0
+
+	for (const byte of bytes) {
+		s += byte
+
+		s ^= s << 13
+		s ^= s >> 17
+		s ^= s << 5
+	}
+
+	return s
+}
+
+export function jenkinsOneAtATime(bytes: Buffer) {
+	let hash = 0
+
+	for (const byte of bytes) {
+		hash += byte
+		hash += hash << 10
+		hash ^= hash >> 6
+	}
+
+	hash += hash << 3
+	hash ^= hash >> 11
+	hash += hash << 15
+
+	return hash >>> 0
+}
+
+export function FNV1a(bytes: Buffer) {
+	let hval = 2166136261 | 0
+
+	for (const byte of bytes) {
+		hval = Math.imul(hval ^ byte, 16777619)
+	}
+
+	return hval >>> 0
+}
+
+export function superFastHash(bytes: Buffer) {
+	let hash = bytes.length, tmp, p = 0
+	const len = bytes.length >>> 2
+
+	for (let i = 0; i < len; i++) {
+		hash += bytes[p] | bytes[p + 1] << 8
+		tmp = ((bytes[p + 2] | bytes[p + 3] << 8) << 11) ^ hash
+		hash = (hash << 16) ^ tmp
+		hash += hash >>> 11
+		p += 4
+	}
+
+	switch (bytes.length & 3) {
+		case 3:
+			hash += bytes[p] | bytes[p + 1] << 8
+			hash ^= hash << 16
+			hash ^= bytes[p + 2] << 18
+			hash += hash >>> 11
+			break
+		case 2:
+			hash += bytes[p] | bytes[p + 1] << 8
+			hash ^= hash << 11
+			hash += hash >>> 17
+			break
+		case 1:
+			hash += bytes[p]
+			hash ^= hash << 10
+			hash += hash >>> 1
+			break
+	}
+
+	hash ^= hash << 3
+	hash += hash >>> 5
+	hash ^= hash << 4
+	hash += hash >>> 17
+	hash ^= hash << 25
+	hash += hash >>> 6
+
+	return hash >>> 0
+}
+
+export function cyrb53Hash(bytes: Buffer, seed = 0) {
+	// https://github.com/bryc/code/blob/master/jshash/experimental/cyrb53.js
+
+	let h1 = 0xdeadbeef ^ seed
+	let h2 = 0x41c6ce57 ^ seed
+
+	for (const byte of bytes) {
+		h1 = Math.imul(h1 ^ byte, 2654435761)
+		h2 = Math.imul(h2 ^ byte, 1597334677)
+	}
+
+	h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909)
+	h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909)
+
+	return (4294967296 * (2097151 & h2)) + (h1 >>> 0)
+}
+
+export function djb2(bytes: Buffer) {
+	let hash = 5381
+
+	for (const byte of bytes) {
+		//hash += (hash << 5) + byte
+		hash += (hash * 33) + byte
+	}
+
+	return hash >>> 0
+}
+
+
+export function murmurHash1(bytes: Buffer, seed = 0) {
+	// https://github.com/bryc/code/blob/master/jshash/hashes/murmurhash1.js
+
+	const length = bytes.length
+
+	const multiplier = 3332679571
+	const intIterationMaxIndex = length & -4
+
+	let hash = seed ^ Math.imul(length, multiplier)
+	let index = 0
+
+	for (; index < intIterationMaxIndex; index += 4) {
+		hash += bytes[index + 3] << 24 |
+			bytes[index + 2] << 16 |
+			bytes[index + 1] << 8 |
+			bytes[index]
+
+		hash = Math.imul(hash, multiplier)
+		hash ^= hash >>> 16
+	}
+
+	switch (length & 3) {
+		case 3: {
+			hash += bytes[index + 2] << 16
+		}
+
+		case 2: {
+			hash += bytes[index + 1] << 8
+		}
+
+		case 1: {
+			hash += bytes[index]
+			hash = Math.imul(hash, multiplier)
+			hash ^= hash >>> 16
+		}
+	}
+
+	hash = Math.imul(hash, multiplier)
+	hash ^= hash >>> 10
+
+	hash = Math.imul(hash, multiplier)
+	hash ^= hash >>> 17
+
+	return hash >>> 0
+}
+
+export function MurmurHash3(bytes: Buffer, seed = 0) {
+	// https://github.com/bryc/code/blob/master/jshash/hashes/murmurhash3.js
+
+	const p1 = 3432918353
+	const p2 = 461845907
+	const p3 = 2246822507
+	const p4 = 3266489909
+
+	const byteCount = bytes.length
+
+	const intIterationMaxIndex = byteCount & -4
+
+	let k = 0
+	let hash = seed | 0
+	let i = 0
+
+	for (; i < intIterationMaxIndex; i += 4) {
+		k = bytes[i + 3] << 24 | bytes[i + 2] << 16 | bytes[i + 1] << 8 | bytes[i]
+
+		k = Math.imul(k, p1)
+
+		k = k << 15 | k >>> 17
+
+		hash ^= Math.imul(k, p2)
+
+		hash = hash << 13 | hash >>> 19
+
+		hash = (Math.imul(hash, 5) + 3864292196) | 0 // |0 = prevent float
+	}
+
+	k = 0
+
+	switch (bytes.length & 3) {
+		case 3: {
+			k ^= bytes[i + 2] << 16
+		}
+
+		case 2: {
+			k ^= bytes[i + 1] << 8
+		}
+
+		case 1: {
+			k ^= bytes[i]
+
+			k = Math.imul(k, p1)
+
+			k = k << 15 | k >>> 17
+
+			hash ^= Math.imul(k, p2)
+		}
+	}
+
+	hash ^= byteCount
+
+	hash ^= hash >>> 16
+
+	hash = Math.imul(hash, p3)
+	hash ^= hash >>> 13
+
+	hash = Math.imul(hash, p4)
+	hash ^= hash >>> 16
+
+	return hash >>> 0
+}

package/src/utilities/Locale.ts

@@ -0,0 +1,119 @@
+import { readAndParseJsonFile, resolveToModuleRootDir } from "./FileSystem.js"
+
+export function languageCodeToName(languageCode: string) {
+	const languageNames = new Intl.DisplayNames(['en'], { type: 'language' })
+
+	let translatedLanguageName: string | undefined
+
+	try {
+		translatedLanguageName = languageNames.of(languageCode)
+	} catch (e) {
+	}
+
+	return translatedLanguageName || "Unknown"
+}
+
+export function formatLanguageCodeWithName(languageCode: string, styleId: 1 | 2 = 1) {
+	if (styleId == 1) {
+		return `${languageCodeToName(languageCode)} (${languageCode})`
+	} else {
+		return `${languageCode}, ${languageCodeToName(languageCode)}`
+	}
+}
+
+export function getShortLanguageCode(langCode: string) {
+	const dashIndex = langCode.indexOf("-")
+
+	if (dashIndex == -1) {
+		return langCode
+	}
+
+	return langCode.substring(0, dashIndex).toLowerCase()
+}
+
+export function normalizeLanguageCode(langCode: string) {
+	langCode = langCode.trim()
+
+	const parts = langCode.split("-")
+
+	const result = [parts[0].toLowerCase()]
+
+	for (let i = 1; i < parts.length; i++) {
+		result.push(parts[i].toUpperCase())
+	}
+
+	return result.join("-")
+}
+
+const isoToLcidLookup = new Map<string, number>()
+const lcidToIsoLookup = new Map<number, string[]>()
+const lcidEntries: LCIDEntry[] = []
+
+export async function isoToLcidLanguageCode(iso: string) {
+	await loadLcidLookupIfNeeded()
+
+	return isoToLcidLookup.get(iso)
+}
+
+export async function lcidToIsoLanguageCode(lcid: number) {
+	await loadLcidLookupIfNeeded()
+
+	return lcidToIsoLookup.get(lcid)
+}
+
+async function loadLcidLookupIfNeeded() {
+	if (lcidEntries.length > 0) {
+		return lcidEntries
+	}
+
+	const lcidLookup: LCIDLookup = await readAndParseJsonFile(resolveToModuleRootDir("data/tables/lcid-table.json"))
+
+	for (const isoName in lcidLookup) {
+		const lcidEntry = lcidLookup[isoName]
+		lcidEntries.push(lcidEntry)
+
+		const lcidValue = lcidEntry.LCID
+
+		isoToLcidLookup.set(isoName, lcidValue)
+
+		let entry = lcidToIsoLookup.get(lcidValue)
+
+		if (!entry) {
+			entry = []
+			lcidToIsoLookup.set(lcidValue, entry)
+		}
+
+		entry.push(isoName)
+	}
+
+	return lcidEntries
+}
+
+export function getDefaultDialectForLanguageCodeIfPossible(langCode: string) {
+	const defaultDialect = defaultDialectForLanguageCode[langCode]
+
+	return defaultDialect || langCode
+}
+
+export const defaultDialectForLanguageCode: { [lang: string]: string } = {
+	"en": "en-US",
+	"zh": "zh-CN",
+	"ar": "ar-EG",
+	"fr": "fr-FR",
+	"de": "de-DE",
+	"pt": "pt-BR",
+	"es": "es-ES",
+	"nl": "nl-NL"
+}
+
+type LCIDLookup = { [isoLangCode: string]: LCIDEntry }
+
+export interface LCIDEntry {
+	"LCID": number
+	"Name": string
+	"TwoLetterISOLanguageName": string,
+	"ThreeLetterISOLanguageName": string,
+	"ThreeLetterWindowsLanguageName": string,
+	"EnglishName": string
+	"ANSICodePage": string
+}

package/src/utilities/Logger.ts

@@ -0,0 +1,72 @@
+import chalk from "chalk"
+import { Timer } from "./Timer.js"
+import { logToStderr, writeToStderr, yieldToEventLoop } from "./Utilities.js"
+
+let currentActiveLogger: Logger | null = null
+
+export class Logger {
+	private timer = new Timer()
+	active = false
+
+	start(title: string, titleColor = chalk.cyanBright) {
+		this.startAsync(title, false, titleColor)
+	}
+
+	async startAsync(title: string, yieldBeforeStart = true, titleColor = chalk.cyanBright) {
+		if (currentActiveLogger != null && currentActiveLogger != this) {
+			return
+		}
+
+		this.end()
+
+		if (yieldBeforeStart) {
+			await yieldToEventLoop()
+		}
+
+		writeToStderr(`${titleColor(title)}.. `)
+		this.active = true
+		currentActiveLogger = this
+		this.timer.restart()
+	}
+
+	end() {
+		if (this.active && currentActiveLogger == this) {
+			const elapsedTime = this.timer.elapsedTime
+
+			writeToStderr(`${elapsedTime.toFixed(1)}ms\n`)
+			currentActiveLogger = null
+		}
+
+		this.active = false
+	}
+
+	logDuration(message: any, startTime: number, titleColor = chalk.cyanBright) {
+		const duration = Timer.currentTime - startTime
+
+		this.log(`${titleColor(message)}: ${duration.toFixed(1)}ms`)
+	}
+
+	logTitledMessage(title: string, content: string, titleColor = chalk.cyanBright) {
+		this.log(`${titleColor(title)}: ${content}`)
+	}
+
+	log(message: any) {
+		if (currentActiveLogger == this || currentActiveLogger == null) {
+			logToStderr(message)
+		}
+	}
+
+	write(message: any) {
+		if (currentActiveLogger == this || currentActiveLogger == null) {
+			writeToStderr(message)
+		}
+	}
+
+	getTimestamp() {
+		return Timer.currentTime
+	}
+}
+
+export function resetActiveLogger() {
+	currentActiveLogger = null
+}

package/src/utilities/NdArrayUtilities.ts

@@ -0,0 +1,31 @@
+import ndarray from 'ndarray'
+import ops from 'ndarray-ops'
+import { medianFilter, softmax } from '../math/VectorMath.js'
+
+export function ndarraySoftMax(vector: ndarray.NdArray, temperature = 1.0) {
+	const vectorAsArray = new Array(vector.shape[0])
+
+	for (let i = 0; i < vectorAsArray.length; i++) {
+		vectorAsArray[i] = vector.get(i)
+	}
+
+	const result = softmax(vectorAsArray, temperature)
+
+	for (let i = 0; i < vectorAsArray.length; i++) {
+		vector.set(i, result[i])
+	}
+}
+
+export function ndarrayMedianFilter(vector: ndarray.NdArray, width: number) {
+	const vectorAsArray = new Array(vector.shape[0])
+
+	for (let i = 0; i < vectorAsArray.length; i++) {
+		vectorAsArray[i] = vector.get(i)
+	}
+
+	const result = medianFilter(vectorAsArray, width)
+
+	for (let i = 0; i < vectorAsArray.length; i++) {
+		vector.set(i, result[i])
+	}
+}