echogarden 0.11.12 → 0.11.13

package/src/utilities/WasmMemoryManager.ts

@@ -0,0 +1,516 @@
+export class WasmMemoryManager {
+	wasmModule: any
+
+	private allocatedReferences = new Set<WasmRef>()
+
+	constructor(wasmModule: any) {
+		this.wasmModule = wasmModule
+	}
+
+	allocInt8() {
+		const address = this.alloc(1)
+		return this.wrapInt8(address).clear()
+	}
+
+	wrapInt8(address: number) {
+		const ref = new Int8Ref(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocUint8() {
+		const address = this.alloc(1)
+		return this.wrapUint8(address).clear()
+	}
+
+	wrapUint8(address: number) {
+		const ref = new Uint8Ref(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocInt16() {
+		const address = this.alloc(2)
+		return this.wrapInt16(address).clear()
+	}
+
+	wrapInt16(address: number) {
+		const ref = new Int16Ref(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocUint16() {
+		const address = this.alloc(2)
+		return this.wrapUint16(address).clear()
+	}
+
+	wrapUint16(address: number) {
+		const ref = new Uint16Ref(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocInt32() {
+		const address = this.alloc(4)
+		return this.wrapInt32(address).clear()
+	}
+
+	wrapInt32(address: number) {
+		const ref = new Int32Ref(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocUint32() {
+		const address = this.alloc(4)
+		return this.wrapUint32(address).clear()
+	}
+
+	wrapUint32(address: number) {
+		const ref = new Uint32Ref(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocPointer() {
+		const address = this.alloc(4)
+		return this.wrapPointer(address).clear()
+	}
+
+	wrapPointer(address: number) {
+		const ref = new PointerRef(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocFloat32() {
+		const address = this.alloc(4)
+		return this.wrapFloat64(address).clear()
+	}
+
+	wrapFloat32(address: number) {
+		const ref = new Float32Ref(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocFloat64() {
+		const address = this.alloc(8)
+		return this.wrapFloat64(address).clear()
+	}
+
+	wrapFloat64(address: number) {
+		const ref = new Float64Ref(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	// Allocate or wrap arrays
+	allocInt8Array(length: number) {
+		const address = this.alloc(length << 0)
+		return this.wrapInt8Array(address, length).clear()
+	}
+
+	wrapInt8Array(address: number, length: number) {
+		const ref = new Int8ArrayRef(address, length, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocUint8Array(length: number) {
+		const address = this.alloc(length << 0)
+		return this.wrapUint8Array(address, length).clear()
+	}
+
+	wrapUint8Array(address: number, length: number) {
+		const ref = new Uint8ArrayRef(address, length, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocInt16Array(length: number) {
+		const address = this.alloc(length << 1)
+		return this.wrapInt16Array(address, length).clear()
+	}
+
+	wrapInt16Array(address: number, length: number) {
+		const ref = new Int16ArrayRef(address, length, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocUint16Array(length: number) {
+		const address = this.alloc(length << 1)
+		return this.wrapUint16Array(address, length).clear()
+	}
+
+	wrapUint16Array(address: number, length: number) {
+		const ref = new Uint16ArrayRef(address, length, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocInt32Array(length: number) {
+		const address = this.alloc(length << 2)
+		return this.wrapInt32Array(address, length).clear()
+	}
+
+	wrapInt32Array(address: number, length: number) {
+		const ref = new Int32ArrayRef(address, length, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocUint32Array(length: number) {
+		const address = this.alloc(length << 2)
+		return this.wrapUint32Array(address, length).clear()
+	}
+
+	wrapUint32Array(address: number, length: number) {
+		const ref = new Uint32ArrayRef(address, length, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocFloat32Array(length: number) {
+		const address = this.alloc(length << 2)
+		return this.wrapFloat32Array(address, length).clear()
+	}
+
+	wrapFloat32Array(address: number, length: number) {
+		const ref = new Float32ArrayRef(address, length, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocFloat64Array(length: number) {
+		const address = this.alloc(length << 3)
+		return this.wrapFloat64Array(address, length).clear()
+	}
+
+	wrapFloat64Array(address: number, length: number) {
+		const ref = new Float64ArrayRef(address, length, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	allocNullTerminatedUtf8String(str: string) {
+		const strBuffer = Buffer.concat([Buffer.from(str, "utf8"), Buffer.alloc(1)])
+		const ref = this.allocUint8Array(strBuffer.length)
+		ref.view.set(strBuffer)
+		return ref
+	}
+
+	wrapNullTerminatedUtf8String(address: number) {
+		const ref = new NullTerminatedUtf8StringRef(address, this)
+		this.allocatedReferences.add(ref)
+		return ref
+	}
+
+	private alloc(size: number) {
+		const ptr = this.wasmModule._malloc(size)
+		return ptr as number
+	}
+
+	free(wasmReference: WasmRef) {
+		if (wasmReference.isFreed) {
+			return
+		}
+
+		this.wasmModule._free(wasmReference.address)
+
+		this.allocatedReferences.delete(wasmReference)
+		wasmReference.clearAddress()
+	}
+
+	freeAll() {
+		for (const wasmReference of this.allocatedReferences) {
+			this.free(wasmReference)
+		}
+	}
+}
+
+abstract class ValueRef<T extends number | string> {
+	protected ptr: number
+	private readonly manager: WasmMemoryManager
+
+	protected get module() { return this.manager.wasmModule }
+
+	constructor(ptr: number, manager: WasmMemoryManager) {
+		this.ptr = ptr
+		this.manager = manager
+	}
+
+	get value(): T {
+		this.assertNotFreed()
+		return this.getValue()
+	}
+
+	set value(newValue: T) {
+		this.assertNotFreed()
+		this.setValue(newValue)
+	}
+
+	abstract getValue(): T
+	abstract setValue(newValue: T): void
+
+	get address() {
+		this.assertNotFreed()
+		return this.ptr
+	}
+
+	clear() {
+		this.assertNotFreed()
+
+		if (typeof this.value == 'number') {
+			this.value = 0 as any
+		} else if (typeof this.value == 'string') {
+			throw new Error("Unimplemented")
+		}
+
+		return this
+	}
+
+	free() {
+		this.manager.free(this as any)
+	}
+
+	clearAddress() {
+		this.ptr = 0
+	}
+
+	get isFreed() { return this.ptr == 0 }
+
+	protected assertNotFreed() {
+		if (this.isFreed) {
+			throw new Error("Attempt to read a freed WASM value reference.")
+		}
+	}
+}
+
+export class Int8Ref extends ValueRef<number> {
+	getValue() {
+		return this.module.HEAP8[this.ptr >>> 0] as number
+	}
+
+	setValue(newValue: number) {
+		this.module.HEAP8[this.ptr >>> 0] = newValue
+	}
+}
+
+export class Uint8Ref extends ValueRef<number> {
+	getValue() {
+		return this.module.HEAPU8[this.ptr >>> 0] as number
+	}
+
+	setValue(newValue: number) {
+		this.module.HEAPU8[this.ptr >>> 0] = newValue
+	}
+}
+
+export class Int16Ref extends ValueRef<number> {
+	getValue() {
+		return this.module.HEAP16[this.ptr >>> 1] as number
+	}
+
+	setValue(newValue: number) {
+		this.module.HEAP16[this.ptr >>> 1] = newValue
+	}
+}
+
+export class Uint16Ref extends ValueRef<number> {
+	getValue() {
+		return this.module.HEAPU16[this.ptr >>> 1] as number
+	}
+
+	setValue(newValue: number) {
+		this.module.HEAPU16[this.ptr >>> 1] = newValue
+	}
+}
+
+export class Int32Ref extends ValueRef<number> {
+	getValue() {
+		return this.module.HEAP32[this.ptr >>> 2] as number
+	}
+
+	setValue(newValue: number) {
+		this.module.HEAP32[this.ptr >>> 2] = newValue
+	}
+}
+
+export class Uint32Ref extends ValueRef<number> {
+	getValue() {
+		return this.module.HEAPU32[this.ptr >>> 2] as number
+	}
+
+	setValue(newValue: number) {
+		this.module.HEAPU32[this.ptr >>> 2] = newValue
+	}
+}
+
+export class PointerRef extends Uint32Ref { }
+
+export class Float32Ref extends ValueRef<number> {
+	getValue() {
+		return this.module.HEAPF32[this.ptr >>> 2] as number
+	}
+
+	setValue(newValue: number) {
+		this.module.HEAPF32[this.ptr >>> 2] = newValue
+	}
+}
+
+export class Float64Ref extends ValueRef<number> {
+	getValue() {
+		return this.module.HEAPF64[this.ptr >>> 3] as number
+	}
+
+	setValue(newValue: number) {
+		this.module.HEAPF64[this.ptr >>> 3] = newValue
+	}
+}
+
+export class NullTerminatedUtf8StringRef extends ValueRef<string> {
+	getValue() {
+		const ptr = this.ptr >>> 0
+
+		const heapU8 = this.module.HEAPU8
+
+		const endByteOffset = heapU8.subarray(ptr).indexOf(0)
+
+		const strBytes = heapU8.subarray(ptr, ptr + endByteOffset)
+
+		const str = Buffer.from(strBytes).toString("utf8")
+
+		return str
+	}
+
+	setValue(newValue: string) {
+		throw new Error("Unimplemented")
+	}
+}
+
+abstract class TypedArrayRef<T extends TypedArray> {
+	protected ptr: number
+	readonly length: number
+	private readonly manager: WasmMemoryManager
+
+	get module() { return this.manager.wasmModule }
+
+	constructor(ptr: number, length: number, manager: WasmMemoryManager) {
+		this.ptr = ptr
+		this.length = length
+		this.manager = manager
+	}
+
+	get view() {
+		this.assertNotFreed()
+		return this.getView()
+	}
+
+	protected abstract getView(): T
+
+	slice(start?: number, end?: number) {
+		return this.view.slice(start, end)
+	}
+
+	get address() {
+		this.assertNotFreed()
+		return this.ptr
+	}
+
+	clear() {
+		this.view.fill(0)
+		return this
+	}
+
+	free() {
+		this.manager.free(this)
+	}
+
+	clearAddress() {
+		this.ptr = 0
+	}
+
+	get isFreed() { return this.ptr == 0 }
+
+	protected assertNotFreed() {
+		if (this.isFreed) {
+			throw new Error("Attempt to read a freed WASM typed array reference.")
+		}
+	}
+}
+
+export class Int8ArrayRef extends TypedArrayRef<Int8Array> {
+	getView() {
+		const startIndex = this.ptr >>> 0
+		return this.module.HEAP8.subarray(startIndex, startIndex + this.length) as Int8Array
+	}
+}
+
+export class Uint8ArrayRef extends TypedArrayRef<Uint8Array> {
+	getView() {
+		const startIndex = this.ptr >>> 0
+		return this.module.HEAPU8.subarray(startIndex, startIndex + this.length) as Uint8Array
+	}
+
+	readAsNullTerminatedUtf8String(): string {
+		let strBytes = this.view
+
+		const indexOfFirstZero = strBytes.indexOf(0)
+
+		if (indexOfFirstZero >= 0) {
+			strBytes = strBytes.subarray(0, indexOfFirstZero)
+		}
+
+		const str = Buffer.from(strBytes).toString("utf8")
+
+		return str
+	}
+}
+
+export class Int16ArrayRef extends TypedArrayRef<Int16Array> {
+	getView() {
+		const startIndex = this.ptr >>> 1
+		return this.module.HEAP16.subarray(startIndex, startIndex + this.length) as Int16Array
+	}
+}
+
+export class Uint16ArrayRef extends TypedArrayRef<Uint16Array> {
+	getView() {
+		const startIndex = this.ptr >>> 1
+		return this.module.HEAPU16.subarray(startIndex, startIndex + this.length) as Uint16Array
+	}
+}
+
+export class Int32ArrayRef extends TypedArrayRef<Int32Array> {
+	getView() {
+		const startIndex = this.ptr >>> 2
+		return this.module.HEAP32.subarray(startIndex, startIndex + this.length) as Int32Array
+	}
+}
+
+export class Uint32ArrayRef extends TypedArrayRef<Uint32Array> {
+	getView() {
+		const startIndex = this.ptr >>> 2
+		return this.module.HEAPU32.subarray(startIndex, startIndex + this.length) as Uint32Array
+	}
+}
+
+export class Float32ArrayRef extends TypedArrayRef<Float32Array> {
+	getView() {
+		const startIndex = this.ptr >>> 2
+		return this.module.HEAPF32.subarray(startIndex, startIndex + this.length) as Float32Array
+	}
+}
+
+export class Float64ArrayRef extends TypedArrayRef<Float64Array> {
+	getView() {
+		const startIndex = this.ptr >>> 3
+		return this.module.HEAPF64.subarray(startIndex, startIndex + this.length) as Float64Array
+	}
+}
+
+export type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array
+export type WasmRef = ValueRef<number> | ValueRef<string> | TypedArrayRef<TypedArray>

package/src/utilities/WebReader.ts

@@ -0,0 +1,55 @@
+import { Readability } from '@mozilla/readability'
+import { JSDOM, VirtualConsole } from 'jsdom'
+import { request } from 'gaxios'
+import { Logger } from './Logger.js'
+import { convertHtmlToText } from './Utilities.js'
+
+export async function fetchDocumentText(url: string) {
+	const progressLogger = new Logger()
+	progressLogger.start(`Fetching ${url}`)
+
+	const response = await request<string>({
+		url,
+		responseType: "text",
+		headers: {
+			"sec-ch-ua": `".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"`,
+			"x-same-domain": "1",
+			"dnt": "1",
+			"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
+			"sec-ch-ua-mobile": "?0",
+			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
+			"sec-ch-ua-arch": "x86",
+			"sec-ch-ua-full-version": "103.0.5060.114",
+			"sec-ch-ua-platform-version": "10.0.0",
+			"sec-ch-ua-full-version-list": `".Not/A)Brand";v="99.0.0.0", "Google Chrome";v="103.0.5060.114", "Chromium";v="103.0.5060.114"`,
+			"sec-ch-ua-bitness": "64",
+			"sec-ch-ua-model": "",
+			"sec-ch-ua-platform": "Windows",
+			"accept": "*/*",
+			//"origin": "https://www.google.com/",
+			"sec-fetch-site": "same-origin",
+			"sec-fetch-mode": "cors",
+			"sec-fetch-dest": "empty",
+			"referer": "https://www.google.com/",
+			"accept-encoding": "gzip, deflate, br",
+			"accept-language": "en-US,en;q=0.9",
+		},
+	})
+
+	progressLogger.start(`Parsing document body`)
+
+	const doc = new JSDOM(response.data, {
+		url,
+		virtualConsole: new VirtualConsole()
+	})
+
+	const reader = new Readability(doc.window.document)
+
+	const article = reader.parse()
+
+	const text: string = await convertHtmlToText(article?.content || "")
+
+	progressLogger.end()
+
+	return text
+}

package/src/utilities/WikipediaReader.ts

@@ -0,0 +1,41 @@
+import { splitToParagraphs, wordCharacterPattern } from '../nlp/Segmentation.js'
+import { Logger } from './Logger.js'
+
+export async function parseWikipediaArticle(articleName: string, language: string) {
+	const logger = new Logger()
+
+	logger.startAsync("Fetching Wikipedia article")
+
+	const { default: wtf } = await import('wtf_wikipedia')
+
+	const document = await wtf.fetch(articleName, language)
+
+	if (!document) {
+		throw new Error("Error fetching Wikipedia article")
+	}
+
+	const sections = document.sections()
+	const sectionsText: string[] = []
+
+	for (const section of sections) {
+		const sectionTitle = section.title()
+
+		if (wordCharacterPattern.test(sectionTitle)) {
+			sectionsText.push(sectionTitle)
+		}
+
+		const sectionParagraphs = splitToParagraphs(section.text(), 'single', 'preserve')
+
+		for (const paragraph of sectionParagraphs) {
+			const paragraphText = paragraph
+
+			if (wordCharacterPattern.test(paragraphText)) {
+				sectionsText.push(paragraphText)
+			}
+		}
+	}
+
+	logger.end()
+
+	return sectionsText
+}

package/src/voice-activity-detection/SileroVAD.ts

@@ -0,0 +1,86 @@
+import Onnx from 'onnxruntime-node'
+
+import { concatFloat32Arrays } from '../utilities/Utilities.js'
+import { RawAudio } from '../audio/AudioUtilities.js'
+
+let sileroVad: SileroVAD
+
+export async function detectVoiceActivity(rawAudio: RawAudio, modelPath: string, frameDuration: 30 | 60 | 90 = 30) {
+	if (rawAudio.sampleRate != 16000) {
+		throw new Error("Audio sample rate must be 16KHz")
+	}
+
+	const audioSamples = rawAudio.audioChannels[0]
+
+	const frameLength = Math.floor(16000 * (frameDuration / 1000))
+
+	if (!sileroVad) {
+		sileroVad = new SileroVAD(modelPath)
+	}
+
+	const frameProbabilities: number[] = []
+
+	for (let position = 0; position < audioSamples.length; position += frameLength) {
+		let chunk = audioSamples.subarray(position, position + frameLength)
+
+		if (chunk.length < frameLength) {
+			chunk = concatFloat32Arrays([chunk, new Float32Array(frameLength - chunk.length)])
+		}
+
+		const probability = await sileroVad.predictAudioFrame(chunk)
+
+		frameProbabilities.push(probability)
+	}
+
+	return frameProbabilities
+}
+
+export class SileroVAD {
+	session?: Onnx.InferenceSession
+
+	modelStateH: Onnx.Tensor
+	modelStateC: Onnx.Tensor
+	modelSampleRate: Onnx.Tensor
+
+	modelPath: string
+
+	constructor(modelPath: string) {
+		this.modelSampleRate = new Onnx.Tensor('int64', new BigInt64Array([BigInt(16000)]), [])
+
+		const h = new Float32Array(2 * 1 * 64)
+		const c = new Float32Array(2 * 1 * 64)
+
+		this.modelStateH = new Onnx.Tensor('float32', h, [2, 1, 64])
+		this.modelStateC = new Onnx.Tensor('float32', c, [2, 1, 64])
+
+		this.modelPath = modelPath
+	}
+
+	async predictAudioFrame(frame: Float32Array) {
+		if (!this.session) {
+			await this.initializeSession(this.modelPath)
+		}
+
+		const inputTensor = new Onnx.Tensor('float32', frame, [1, frame.length])
+
+		//const inputs = { input: inputTensor, h0: this.modelStateH, c0: this.modelStateC }
+		const inputs = { input: inputTensor, sr: this.modelSampleRate, h: this.modelStateH, c: this.modelStateC }
+
+		const results = await this.session!.run(inputs)
+
+		const probability = results["output"].data[0] as number
+
+		this.modelStateH = results["hn"]
+		this.modelStateC = results["cn"]
+
+		return probability
+	}
+
+	private async initializeSession(modelPath: string) {
+		const onnxOptions: Onnx.InferenceSession.SessionOptions = {
+			logSeverityLevel: 3
+		}
+
+		this.session = await Onnx.InferenceSession.create(modelPath, onnxOptions)
+	}
+}