echogarden 0.11.11 → 0.11.13

package/src/utilities/ObjectUtilities.ts

@@ -0,0 +1,169 @@
+export function extendDeep(base: any, extension: any): any {
+	const baseClone = deepClone(base)
+
+	if (isPlainObject(base) && extension === undefined) {
+		return baseClone
+	}
+
+	const extensionClone = deepClone(extension)
+	if (!isPlainObject(base) || !isPlainObject(extension)) {
+		return extensionClone
+	}
+
+	for (const propName in extensionClone) {
+		if (!extensionClone.hasOwnProperty(propName)) {
+			continue
+		}
+
+		baseClone[propName] = extendDeep(baseClone[propName], extensionClone[propName])
+	}
+
+	return baseClone
+}
+
+export function shallowClone<T>(val: T) {
+	return clone(val, false)
+}
+
+export function deepClone<T>(val: T) {
+	return clone(val, true)
+}
+
+function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
+	if (val == null || typeof val !== "object") {
+		return val
+	}
+
+	const obj = <any>val
+	const prototypeIdentifier = toString.call(obj)
+
+	switch (prototypeIdentifier) {
+		case "[object Array]": {
+			if (seenObjects.includes(obj)) {
+				throw new Error("deepClone: encountered a cyclic object")
+			}
+
+			seenObjects.push(obj)
+
+			const clonedArray = new Array(obj.length)
+
+			for (let i = 0; i < obj.length; i++) {
+				if (deep) {
+					clonedArray[i] = clone(obj[i], true, seenObjects)
+				} else {
+					clonedArray[i] = obj[i]
+				}
+			}
+
+			seenObjects.pop()
+
+			return <any>clonedArray
+		}
+
+		case "[object ArrayBuffer]": {
+			const clonedArray = new Uint8Array(obj.byteLength)
+			clonedArray.set(new Uint8Array(obj))
+			return <any>clonedArray.buffer
+		}
+
+		case "[object Int8Array]": {
+			const clonedArray = new Int8Array(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Uint8Array]": {
+			const clonedArray = new Uint8Array(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Uint8ClampedArray]": {
+			const clonedArray = new Uint8ClampedArray(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Int16Array]": {
+			const clonedArray = new Int16Array(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Uint16Array]": {
+			const clonedArray = new Uint16Array(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Int32Array]": {
+			const clonedArray = new Int32Array(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Uint32Array]": {
+			const clonedArray = new Uint32Array(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Float32Array]": {
+			const clonedArray = new Float32Array(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Float64Array]": {
+			const clonedArray = new Float64Array(obj.length)
+			clonedArray.set(obj)
+			return <any>clonedArray
+		}
+
+		case "[object Date]": {
+			return <any>new Date(obj.valueOf())
+		}
+
+		case "[object RegExp]": {
+			return obj
+		}
+
+		case "[object Function]": {
+			return obj
+		}
+
+		case "[object Object]": {
+			if (seenObjects.includes(obj)) {
+				throw new Error("deepClone: encountered a cyclic object")
+			}
+
+			seenObjects.push(obj)
+
+			const clonedObj: any = {}
+
+			for (const propName in obj) {
+				if (!obj.hasOwnProperty(propName)) {
+					continue
+				}
+
+				if (deep) {
+					clonedObj[propName] = clone(obj[propName], true, seenObjects)
+				} else {
+					clonedObj[propName] = obj[propName]
+				}
+			}
+
+			seenObjects.pop()
+
+			return clonedObj
+		}
+
+		default: {
+			throw new Error(`Cloning of type ${prototypeIdentifier} is not supported`)
+		}
+	}
+}
+
+export function isPlainObject(val: any) {
+	return val != null && typeof val === "object" && toString.call(val) === "[object Object]"
+}

package/src/utilities/OpenPromise.ts

@@ -0,0 +1,13 @@
+export class OpenPromise<T = void>
+{
+	promise: Promise<T>
+	resolve: (value: T) => void = () => { throw new Error("OpenPromise resolved before initialization") }
+	reject: (reason?: any) => void = () => { throw new Error("OpenPromise rejected before initialization") }
+
+	constructor() {
+		this.promise = new Promise<T>((resolve, reject) => {
+			this.resolve = resolve
+			this.reject = reject
+		})
+	}
+}

package/src/utilities/PackageManager.ts

@@ -0,0 +1,97 @@
+import path from "node:path"
+import { downloadAndExtractTarball } from "./FileDownloader.js"
+import { getAppDataDir, getAppTempDir, ensureDir, existsSync, remove } from "./FileSystem.js"
+import { appName } from "../api/Common.js"
+import { GaxiosOptions } from "gaxios"
+
+export async function loadPackage(packageName: string) {
+	packageName = resolveToVersionedPackageNameIfNeeded(packageName)
+
+	const packagesPath = await ensureAndGetPackagesDir()
+
+	const packagePath = path.join(packagesPath, packageName)
+
+	if (existsSync(packagePath)) {
+		return packagePath
+	}
+
+	const tempPath = getAppTempDir(appName)
+
+	const headers = {
+	}
+
+	const options: GaxiosOptions = {
+		url: `${basePackageUrl}${packageName}.tar.gz`,
+		headers
+	}
+
+	await downloadAndExtractTarball(
+		options,
+		packagesPath,
+		tempPath,
+		packageName,
+	)
+
+	return packagePath
+}
+
+export async function removePackage(packageName: string) {
+	packageName = resolveToVersionedPackageNameIfNeeded(packageName)
+
+	const packagesPath = await ensureAndGetPackagesDir()
+
+	const packagePath = path.join(packagesPath, packageName)
+
+	await remove(packagePath)
+}
+
+export async function ensureAndGetPackagesDir() {
+	const dataPath = getAppDataDir(appName)
+
+	const packagesPath = path.join(dataPath, "packages")
+
+	await ensureDir(packagesPath)
+
+	return packagesPath
+}
+
+export function resolveToVersionedPackageNameIfNeeded(packageName: string) {
+	const versionTag = getVersionTagFromPackageName(packageName)
+
+	if (versionTag) {
+		return packageName
+	}
+
+	const resolvedVersionTag = resolveVersionTagForUnversionedPackageName(packageName)
+
+	return packageName = `${packageName}-${resolvedVersionTag}`
+}
+
+export function getVersionTagFromPackageName(packageName: string) {
+	return packageName.match(/.*\-([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9](_[0-9]+)?)$/)?.[1]
+}
+
+export function resolveVersionTagForUnversionedPackageName(unversionedPackageName: string) {
+	return packageVersionTagResolutionLookup[unversionedPackageName] || defaultVersionTag
+}
+
+const basePackageUrl = "https://huggingface.co/echogarden/echogarden-packages/resolve/main/"
+
+const defaultVersionTag = "20230718"
+
+const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
+	"sox-14.4.2-linux-minimal": "20230802",
+	"vits-de_DE-thorsten_emotional-medium": "20230808",
+	"vits-en_GB-semaine-medium": "20230808",
+	"vits-fr_FR-upmc-medium": "20230808",
+	"vits-lb_LU-marylux-medium": "20230808",
+	"vits-ro_RO-mihai-medium": "20230808",
+	"vits-sr_RS-serbski_institut-medium": "20230808",
+	"vits-tr_TR-dfki-medium": "20230808",
+	"vits-cs_CZ-jirka-medium": "20230824",
+	"vits-de_DE-thorsten-high": "20230824",
+	"vits-hu_HU-anna-medium": "20230824",
+	"vits-pt_PT-tugao-medium": "20230824",
+	"vits-sk_SK-lili-medium": "20230824",
+	"vits-tr_TR-fahrettin-medium": "20230824",
+}

package/src/utilities/Queue.ts

@@ -0,0 +1,17 @@
+export class Queue<T> {
+	list: T[] = []
+
+	constructor() {
+	}
+
+	enqueue(item: T) {
+		this.list.push(item)
+	}
+
+	dequeue(): T | undefined {
+		return this.list.shift()
+	}
+
+	get isEmpty() { return this.list.length == 0 }
+	get isNonempty() { return !this.isEmpty }
+}

package/src/utilities/RandomGenerator.ts

@@ -0,0 +1,237 @@
+import { sumVector } from "../math/VectorMath.js"
+import { logToStderr } from "./Utilities.js"
+
+export abstract class RandomGenerator {
+	getIntInRange(min: number, max: number) {
+		return Math.floor(this.getFloatInRange(min, max))
+	}
+
+	getFloatInRange(min = 0, max = 1) {
+		return min + (this.nextFloat() * (max - min))
+	}
+
+	getFloats(length: number, min = 0, max = 1) {
+		const result = []
+
+		for (let i = 0; i < length; i++) {
+			result.push(this.getFloatInRange(min, max))
+		}
+
+		return result
+	}
+
+	getInts(length: number, min: number, max: number) {
+		const result = []
+
+		for (let i = 0; i < length; i++) {
+			result.push(this.getIntInRange(min, max))
+		}
+
+		return result
+	}
+
+	getNormallyDistributedVector(featureCount: number, meanVector: number[], standardDeviationVector: number[]) {
+		const features = this.getNormallyDistributedValues(featureCount)
+
+		for (let i = 0; i < features.length; i++) {
+			features[i] = meanVector[i] + (features[i] * standardDeviationVector[i])
+		}
+
+		return features
+	}
+
+	getNormallyDistributedValues(count: number, mean = 0, standardDeviation = 1) {
+		const result: number[] = []
+
+		for (let i = 0; i < count; i += 2) {
+			const [n1, n2] = this.getNormallyDistributedPair()
+
+			result.push(mean + (n1 * standardDeviation))
+
+			if (i + 1 < count) {
+				result.push(mean + (n2 * standardDeviation))
+			}
+		}
+
+		return result
+	}
+
+	getNormallyDistributedPair() {
+		// Using Marsaglia polar method
+		// https://en.wikipedia.org/wiki/Marsaglia_polar_method
+
+		let x: number
+		let y: number
+		let s: number
+
+		do {
+			x = (this.nextFloat() * 2) - 1
+			y = (this.nextFloat() * 2) - 1
+
+			s = (x ** 2) + (y ** 2)
+		} while (s <= 0 || s >= 1)
+
+		const m = Math.sqrt((-2 * Math.log(s)) / s)
+
+		const n1 = x * m
+		const n2 = y * m
+
+		return [n1, n2]
+	}
+
+	selectRandomIndexFromDistribution(distribution: number[]) {
+		const sum = sumVector(distribution)
+
+		const randomTarget = this.getFloatInRange(0, sum)
+
+		let cumSum = 0
+		
+		for (let i = 0; i < distribution.length; i++) {
+			const element = distribution[i]
+
+			if (randomTarget < cumSum + element) {
+				return i
+			}
+
+			cumSum += element
+		}
+
+		return distribution.length - 1
+	}
+
+	abstract nextFloat(): number
+	abstract nextUint32(): number
+	abstract nextInt32(): number
+}
+
+export class MurmurRNG extends RandomGenerator {
+	state: number
+
+	constructor(seed: number) {
+		super()
+
+		this.state = seed
+	}
+
+	nextFloat() {
+		return this.nextUint32() / 4294967296
+	}
+
+	nextUint32() {
+		return this.nextInt32() >>> 0
+	}
+
+	nextInt32() {
+		const multiplier = 3332679571
+
+		let s = this.state
+
+		s = Math.imul(s, multiplier)
+		s ^= s >>> 16
+
+		s = Math.imul(s, multiplier)
+		s ^= s >>> 10
+
+		s = Math.imul(s, multiplier)
+		s ^= s >>> 17
+
+		this.state = s
+
+		return s
+	}
+
+	static hashInt32(val: number, seed = 234928357) {
+		const rng = new MurmurRNG(seed ^ val)
+		return rng.nextUint32()
+	}
+}
+
+export class XorShift32RNG extends RandomGenerator {
+	state: number
+
+	constructor(seed: number) {
+		super()
+
+		this.state = seed
+	}
+
+	nextFloat() {
+		return this.nextUint32() / 4294967296
+	}
+
+	nextUint32() {
+		return this.nextInt32() >>> 0
+	}
+
+	nextInt32() {
+		let s = this.state
+
+		s ^= s << 13
+		s ^= s >>> 17
+		s ^= s << 5
+
+		this.state = s
+
+		return s
+	}
+}
+
+export class LehmerRNG extends RandomGenerator {
+	state: number
+
+	constructor(seed: number) {
+		super()
+
+		this.state = seed
+	}
+
+	nextFloat() {
+		return this.nextUint32() / 4294967296
+	}
+
+	nextUint32() {
+		return this.nextInt32() >>> 0
+	}
+
+	nextInt32() {
+		let s = this.state
+
+		s = s * 48271 % 0x7fffffff
+
+		this.state = s
+
+		return s
+	}
+}
+
+const log = logToStderr
+
+export function testRngCycleLength() {
+	const seed = 1
+
+	const rng = new XorShift32RNG(seed)
+
+	const bucketCount = 1000000
+	const seen: number[][] = []
+
+	for (let i = 0; i < bucketCount; i++) {
+		seen[i] = []
+	}
+
+	for (let i = 0; i < 4294967296; i++) {
+		if (i % 1000000 == 0) {
+			log(`${i} iterations`)
+		}
+
+		const r = rng.nextUint32()
+
+		const bucket = seen[r % bucketCount]
+
+		if (bucket.includes(r)) {
+			log(`Cycles at iteration ${i}`)
+			return
+		}
+
+		bucket.push(r)
+	}
+}

package/src/utilities/SignalChannel.ts

@@ -0,0 +1,22 @@
+export class SignalChannel {
+	channel: MessageChannel
+
+	constructor() {
+		this.channel = new MessageChannel()
+
+	}
+
+	on(signalName: string, handler: (data?: any) => void) {
+		this.channel.port2.onmessage = (event: MessageEvent) => {
+			if (event.data.signalName != signalName) {
+				return
+			}
+
+			handler(event.data.data)
+		}
+	}
+
+	send(signalName: string, data?: any) {
+		this.channel.port1.postMessage({ signalName, data })
+	}
+}

package/src/utilities/TarballMaker.ts

@@ -0,0 +1,68 @@
+import path from "path"
+import { createTarball } from "./Compression.js"
+import { ensureDir, existsSync, getAppTempDir, move, readdir, stat } from "./FileSystem.js"
+import { ensureAndGetPackagesDir } from "./PackageManager.js"
+import { appName } from "../api/Common.js"
+import { getRandomHexString } from "./Utilities.js"
+
+const tarballDir = "../resources/tarballs"
+
+export async function makeTarballsForInstalledPackages(skipIfExists = false) {
+	const packagesDir = await ensureAndGetPackagesDir()
+	const packageList = await readdir(packagesDir)
+
+	for (const packageName of packageList) {
+		if (skipIfExists && existsSync(path.join(tarballDir, `${packageName}.tar.gz`))) {
+			continue
+		}
+
+		const packagePath = path.join(packagesDir, packageName)
+		await createNamedTarball(packagePath, packageName)
+	}
+}
+
+export async function createNamedTarball(inputPath: string, name: string) {
+	const tempDir = getAppTempDir(appName)
+	await ensureDir(tempDir)
+	const tempFilename = path.join(tempDir, getRandomHexString(16))
+
+	await createTarball(inputPath, tempFilename, name)
+
+	const targetFilname = path.join(tarballDir, `${name}.tar.gz`)
+
+	await move(tempFilename, targetFilname)
+}
+
+export async function createTarballForEachDirIn(baseDir: string, namePrefix: String) {
+	for (const dirName of await readdir(baseDir)) {
+		const dirPath = path.join(baseDir, dirName)
+
+		const fileStat = await stat(dirPath)
+
+		if (!fileStat.isDirectory()) {
+			continue
+		}
+
+		const archiveName = `${namePrefix}-${dirName}`
+
+		await createNamedTarball(dirPath, archiveName)
+	}
+}
+
+export async function createTarballForEachFileIn(baseDir: string, namePrefix: String) {
+	for (const filename of await readdir(baseDir)) {
+		const filenameWithoutExtension = path.parse(filename).name
+
+		const filePath = path.join(baseDir, filename)
+
+		const fileStat = await stat(filePath)
+
+		if (!fileStat.isFile()) {
+			continue
+		}
+
+		const archiveName = `${namePrefix}-${filenameWithoutExtension}`
+
+		await createNamedTarball(filePath, archiveName)
+	}
+}