echogarden 1.8.7 → 2.0.0

package/src/utilities/BinaryArrayConversion.ts

@@ -1,186 +1,112 @@
-// Typed arrays to Buffer (little endian) conversions
+// Typed arrays to Uint8Array buffers (little endian) conversions
 //
-// The faster conversion methods (other than the methods for int8) would only work correctly
+// The conversion methods (other than the methods for int8 and int24) would only work correctly
 // on little-endian architectures, since they assume the byte order of the underlying architecture.
 //
 // Since Echogarden only supports little-endian architectures, this shouldn't matter.
 
 // int8 <-> bufferLE
 export function int8ToBuffer(int8s: Int8Array) {
-	return Buffer.copyBytesFrom(int8s)
+	return new Uint8Array(int8s.buffer, int8s.byteOffset, int8s.byteLength)
 }
 
-export function int8ToBuffer_Slow(int8s: Int8Array) {
-	const buffer = Buffer.alloc(int8s.length)
-
-	for (let i = 0; i < int8s.length; i++) {
-		buffer[i] = int8s[i] + 128
-	}
-
-	return buffer
-}
-
-export function bufferToInt8(buffer: Buffer) {
+export function bufferToInt8(buffer: Uint8Array) {
 	return new Int8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
 }
 
-export function bufferToInt8_Slow(buffer: Buffer) {
-	const result = new Int8Array(buffer.length)
-
-	for (let i = 0; i < result.length; i++) {
-		result[i] = buffer[i] - 128
-	}
-
-	return result
-}
-
 // int16 <-> bufferLE
 export function int16ToBufferLE(int16s: Int16Array) {
-	return Buffer.copyBytesFrom(int16s)
-}
-
-export function int16ToBufferLE_Slow(int16s: Int16Array) {
-	const buffer = Buffer.alloc(int16s.length * 2)
-
-	for (let i = 0; i < int16s.length; i++) {
-		buffer.writeInt16LE(int16s[i], i * 2)
-	}
-
-	return buffer
+	return new Uint8Array(int16s.buffer, int16s.byteOffset, int16s.byteLength)
 }
 
-export function bufferLEToInt16(buffer: Buffer) {
+export function bufferLEToInt16(buffer: Uint8Array) {
 	return new Int16Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 2)
 }
 
-export function bufferLEToInt16_Slow(buffer: Buffer) {
-	const result = new Int16Array(buffer.length / 2)
+// int24 <-> bufferLE (uses int32 for storage)
+export function int24ToBufferLE(int24s: Int32Array) {
+	const buffer = new Uint8Array(int24s.length * 3)
 
-	for (let i = 0; i < result.length; i++) {
-		result[i] = buffer.readInt16LE(i * 2)
-	}
+	let readOffset = 0
+	let writeOffset = 0
 
-	return result
-}
+	while (readOffset < int24s.length) {
+		const signedValue = int24s[readOffset++]
 
-// int24 <-> bufferLE (uses int32 for storage)
-export function int24ToBufferLE(int24s: Int32Array) {
-	const buffer = Buffer.alloc(int24s.length * 3)
+		let unsignedValue: number
 
-	for (let i = 0; i < int24s.length; i++) {
-		const val = int24s[i]
-		const encodedVal = val < 0 ? val + 0x1000000 : val
+		if (signedValue >= 0) {
+			unsignedValue = signedValue
+		} else {
+			unsignedValue = signedValue + (2 ** 24)
+		}
 
-		buffer[(i * 3) + 0] = (encodedVal >> 0) & 0xff
-		buffer[(i * 3) + 1] = (encodedVal >> 8) & 0xff
-		buffer[(i * 3) + 2] = (encodedVal >> 16) & 0xff
+		buffer[writeOffset++] = (unsignedValue) & 0xff
+		buffer[writeOffset++] = (unsignedValue >> 8) & 0xff
+		buffer[writeOffset++] = (unsignedValue >> 16) & 0xff
 	}
 
 	return buffer
 }
 
-export function bufferLEToInt24(buffer: Buffer) {
+export function bufferLEToInt24(buffer: Uint8Array) {
+	if (buffer.length % 3 !== 0) {
+		throw new Error(`Buffer has a length of ${buffer.length}, which is not a multiple of 3`)
+	}
+
 	const result = new Int32Array(buffer.length / 3)
 
-	for (let i = 0; i < result.length; i++) {
-		const b0 = buffer[(i * 3) + 0]
-		const b1 = buffer[(i * 3) + 1]
-		const b2 = buffer[(i * 3) + 2]
+	let readOffset = 0
+	let writeOffset = 0
 
-		const encodedVal = (b0 << 0) + (b1 << 8) + (b2 << 16)
-		result[i] = encodedVal > 0x800000 ? encodedVal - 0x1000000 : encodedVal
-	}
+	while (writeOffset < result.length) {
+		const b0 = buffer[readOffset++]
+		const b1 = buffer[readOffset++]
+		const b2 = buffer[readOffset++]
 
-	return result
-}
+		const unsignedValue = (b0) | (b1 << 8) | (b2 << 16)
 
-// int32 <-> bufferLE
-export function int32ToBufferLE(int32s: Int32Array) {
-	return Buffer.copyBytesFrom(int32s)
-}
+		let signedValue: number
 
-export function int32ToBufferLE_Slow(int32s: Int32Array) {
-	const buffer = Buffer.alloc(int32s.length * 4)
+		if (unsignedValue < 2 ** 23) {
+			signedValue = unsignedValue
+		} else {
+			signedValue = unsignedValue - (2 ** 24)
+		}
 
-	for (let i = 0; i < int32s.length; i++) {
-		buffer.writeInt32LE(int32s[i], i * 4)
+		result[writeOffset++] = signedValue
 	}
 
-	return buffer
+	return result
 }
 
-export function bufferLEToInt32(buffer: Buffer) {
-	return new Int32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
+// int32 <-> bufferLE
+export function int32ToBufferLE(int32s: Int32Array) {
+	return new Uint8Array(int32s.buffer, int32s.byteOffset, int32s.byteLength)
 }
 
-export function bufferLEToInt32_Slow(buffer: Buffer) {
-	const result = new Int32Array(buffer.length / 4)
-
-	for (let i = 0; i < result.length; i++) {
-		result[i] = buffer.readInt32LE(i * 4)
-	}
-
-	return result
+export function bufferLEToInt32(buffer: Uint8Array) {
+	return new Int32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
 }
 
 // float32 <-> bufferLE
 export function float32ToBufferLE(float32s: Float32Array) {
-	return Buffer.copyBytesFrom(float32s)
+	return new Uint8Array(float32s.buffer, float32s.byteOffset, float32s.byteLength)
 }
 
-export function float32ToBufferLE_Slow(float32s: Float32Array) {
-	const buffer = Buffer.alloc(float32s.length * 4)
-
-	for (let i = 0; i < float32s.length; i++) {
-		buffer.writeFloatLE(float32s[i], i * 4)
-	}
-
-	return buffer
-}
-
-export function bufferLEToFloat32(buffer: Buffer) {
+export function bufferLEToFloat32(buffer: Uint8Array) {
 	return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
 }
 
-export function bufferLEToFloat32_Slow(buffer: Buffer) {
-	const result = new Float32Array(buffer.length / 4)
-
-	for (let i = 0; i < result.length; i++) {
-		result[i] = buffer.readFloatLE(i * 4)
-	}
-
-	return result
-}
-
 // float64 <-> bufferLE
 export function float64ToBufferLE(float64s: Float64Array) {
-	return Buffer.copyBytesFrom(float64s)
-}
-
-export function float64ToBufferLE_Slow(float64s: Float64Array) {
-	const buffer = Buffer.alloc(float64s.length * 8)
-
-	for (let i = 0; i < float64s.length; i++) {
-		buffer.writeDoubleLE(float64s[i], i * 8)
-	}
-
-	return buffer
+	return new Uint8Array(float64s.buffer, float64s.byteOffset, float64s.byteLength)
 }
 
-export function bufferLEToFloat64(buffer: Buffer) {
+export function bufferLEToFloat64(buffer: Uint8Array) {
 	return new Float64Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 8)
 }
 
-export function bufferLEToFloat64_Slow(buffer: Buffer) {
-	const result = new Float64Array(buffer.length / 8)
-
-	for (let i = 0; i < result.length; i++) {
-		result[i] = buffer.readDoubleLE(i * 8)
-	}
-
-	return result
-}
-
 // float64 <-> float32
 export function float64Tofloat32(float64s: Float64Array) {
 	return Float32Array.from(float64s)

package/src/utilities/BinaryUtilities.ts

@@ -0,0 +1,138 @@
+/////////////////////////////////////////////////////////////////////////////////////
+// Unsigned operations
+/////////////////////////////////////////////////////////////////////////////////////
+export function readUint8(buffer: Uint8Array, offset: number) {
+	return buffer[offset]
+}
+
+export function writeUint8(buffer: Uint8Array, value: number, offset: number) {
+	if (value < 0 || value > 255) {
+		throw new Error(`Value ${value} is outside the range of an 8-bit unsigned integer`)
+	}
+
+	buffer[offset] = value
+}
+
+export function readUint16LE(buffer: Uint8Array, offset: number) {
+	return (buffer[offset]) | (buffer[offset + 1] << 8)
+}
+
+export function writeUint16LE(buffer: Uint8Array, value: number, offset: number) {
+	if (value < 0 || value > 65535) {
+		throw new Error(`Value ${value} is outside the range of a 16-bit unsigned integer`)
+	}
+
+	buffer[offset] = value & 0xff
+	buffer[offset + 1] = (value >>> 8) & 0xff
+}
+
+export function readUint32LE(buffer: Uint8Array, offset: number) {
+	return readInt32LE(buffer, offset) >>> 0
+}
+
+export function writeUint32LE(buffer: Uint8Array, value: number, offset: number) {
+	if (value < 0 || value > 4294967295) {
+		throw new Error(`Value ${value} is outside the range of a 32-bit unsigned integer`)
+	}
+
+	buffer[offset] = value & 0xff
+	buffer[offset + 1] = (value >>> 8) & 0xff
+	buffer[offset + 2] = (value >>> 16) & 0xff
+	buffer[offset + 3] = (value >>> 24) & 0xff
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+// Signed operations
+/////////////////////////////////////////////////////////////////////////////////////
+export function readInt8(buffer: Uint8Array, offset: number) {
+	const unsignedValue = buffer[offset]
+
+	if (unsignedValue < 128) {
+		return unsignedValue
+	} else {
+		return unsignedValue - 256
+	}
+}
+
+export function writeInt8(buffer: Uint8Array, value: number, offset: number) {
+	if (value < -128 || value > 127) {
+		throw new Error(`Value ${value} is outside the range of an 8-bit signed integer`)
+	}
+
+	let unsignedValue: number
+
+	if (value >= 0) {
+		unsignedValue = value
+	} else {
+		unsignedValue = value + 256
+	}
+
+	buffer[offset] = unsignedValue
+}
+
+export function readInt16LE(buffer: Uint8Array, offset: number) {
+	const unsignedValue = readUint16LE(buffer, offset)
+
+	if (unsignedValue < 32768) {
+		return unsignedValue
+	} else {
+		return unsignedValue - 65536
+	}
+}
+
+export function writeInt16LE(buffer: Uint8Array, value: number, offset: number) {
+	if (value < -32768 || value > 32767) {
+		throw new Error(`Value ${value} is outside the range of a 16-bit signed integer`)
+	}
+
+	let unsignedValue: number
+
+	if (value >= 0) {
+		unsignedValue = value
+	} else {
+		unsignedValue = value + 65536
+	}
+
+	writeUint16LE(buffer, unsignedValue, offset)
+}
+
+export function readInt32LE(buffer: Uint8Array, offset: number) {
+	const value =
+		(buffer[offset]) |
+		(buffer[offset + 1] << 8) |
+		(buffer[offset + 2] << 16) |
+		(buffer[offset + 3] << 24)
+
+	return value
+}
+
+export function writeInt32LE(buffer: Uint8Array, value: number, offset: number) {
+	if (value < -2147483648 || value > 2147483647) {
+		throw new Error(`Value ${value} is outside the range of a 32-bit signed integer`)
+	}
+
+	buffer[offset] = value & 0xff
+	buffer[offset + 1] = (value >> 8) & 0xff
+	buffer[offset + 2] = (value >> 16) & 0xff
+	buffer[offset + 3] = (value >> 24) & 0xff
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+// Misc operations
+/////////////////////////////////////////////////////////////////////////////////////
+export function writeAscii(buffer: Uint8Array, asciiString: string, writeStartOffset: number) {
+	const writeEndOffset = Math.min(writeStartOffset + asciiString.length, buffer.length)
+
+	let readOffset = 0
+	let writeOffset = writeStartOffset
+
+	while (writeOffset < writeEndOffset) {
+		const charCode = asciiString.charCodeAt(readOffset++)
+
+		if (charCode >= 128) {
+			throw new Error(`Character '${asciiString[readOffset]}' (code: ${charCode}) can't be encoded as ASCII`)
+		}
+
+		buffer[writeOffset++] = charCode
+	}
+}

package/src/utilities/Compression.ts

@@ -1,6 +1,7 @@
-import path from 'node:path'
 import { Logger } from './Logger.js'
 import { readdir, stat } from './FileSystem.js'
+import { encodeUtf8 } from '../encodings/Utf8.js'
+import { getBaseName, getDirName } from './PathUtilities.js'
 
 export async function createTarball(filePath: string, outputFile: string, prefixPath = '') {
 	const pathStat = await stat(filePath)
@@ -15,7 +16,7 @@ export async function createTarball(filePath: string, outputFile: string, prefix
 export async function createTarballForFile(filePath: string, outputFile: string, prefixPath = '') {
 	const logger = new Logger()
 
-	logger.start(`Creating ${prefixPath || path.basename(outputFile)}`)
+	logger.start(`Creating ${prefixPath || getBaseName(outputFile)}`)
 
 	const { create } = await import('tar')
 
@@ -25,8 +26,8 @@ export async function createTarballForFile(filePath: string, outputFile: string,
 		throw new Error(`${filePath} is not a file`)
 	}
 
-	const filename = path.basename(filePath)
-	const dirname = path.dirname(filePath)
+	const filename = getBaseName(filePath)
+	const dirname = getDirName(filePath)
 
 	await create({
 		gzip: { level: 9 },
@@ -51,7 +52,7 @@ export async function createTarballForFile(filePath: string, outputFile: string,
 export async function createTarballForDir(inputDir: string, outputFile: string, prefixPath = '') {
 	const logger = new Logger()
 
-	logger.start(`Creating ${prefixPath || path.basename(outputFile)}`)
+	logger.start(`Creating ${prefixPath || getBaseName(outputFile)}`)
 
 	const { create } = await import('tar')
 
@@ -98,7 +99,7 @@ export async function getDeflateCompressionMetricsForString(str: string) {
 	const zlib = await import('node:zlib')
 	const { promisify } = await import('node:util')
 
-	const originalStringBytes = Buffer.from(str, 'utf-8')
+	const originalStringBytes = encodeUtf8(str)
 
 	const deflateRaw = promisify(zlib.deflateRaw)
 

package/src/utilities/FileDownloader.ts

@@ -1,22 +1,22 @@
 import { GaxiosOptions, request } from 'gaxios'
 import { Readable } from 'stream'
 import { getRandomHexString, writeToStderr } from './Utilities.js'
-import { OpenPromise } from './OpenPromise.js'
 
 import { Timer } from './Timer.js'
 import { Logger } from './Logger.js'
-import path from 'node:path'
 import { extractTarball } from './Compression.js'
-import { createWriteStream, move, remove, readdir, ensureDir } from './FileSystem.js'
+import { move, remove, readdir, ensureDir } from './FileSystem.js'
 import chalk from 'chalk'
 import { logLevelGreaterOrEqualTo } from '../api/GlobalOptions.js'
+import { FileWriter } from './FileWriter.js'
+import { joinPath } from './PathUtilities.js'
 
 export async function downloadAndExtractTarball(options: GaxiosOptions, targetDir: string, baseTempPath: string, displayName = 'archive') {
 	const logger = new Logger()
 
 	const randomID = getRandomHexString(16).toLowerCase()
-	const tempTarballPath = path.join(baseTempPath, `/${randomID}.tarball`)
-	const tempDirPath = path.join(baseTempPath, `/${randomID}`)
+	const tempTarballPath = joinPath(baseTempPath, `/${randomID}.tarball`)
+	const tempDirPath = joinPath(baseTempPath, `/${randomID}`)
 	await ensureDir(tempDirPath)
 
 	logger.end()
@@ -32,8 +32,8 @@ export async function downloadAndExtractTarball(options: GaxiosOptions, targetDi
 	await remove(tempTarballPath)
 
 	for (const filename of await readdir(tempDirPath)) {
-		const sourceFilePath = path.join(tempDirPath, filename)
-		const targetFilePath = path.join(targetDir, filename)
+		const sourceFilePath = joinPath(tempDirPath, filename)
+		const targetFilePath = joinPath(targetDir, filename)
 
 		await move(sourceFilePath, targetFilePath)
 	}
@@ -46,8 +46,6 @@ export async function downloadAndExtractTarball(options: GaxiosOptions, targetDi
 export async function downloadFile(options: GaxiosOptions, targetFilePath: string, prompt = 'Downloading') {
 	const write = logLevelGreaterOrEqualTo('info') ? writeToStderr : () => {}
 
-	const downloadPromise = new OpenPromise<void>()
-
 	const timer = new Timer()
 
 	options.responseType = 'stream'
@@ -143,63 +141,40 @@ export async function downloadFile(options: GaxiosOptions, targetFilePath: strin
 		}
 	}
 
+	const contentLengthString = response.headers['content-length']
+	totalBytes = contentLengthString != undefined ? parseInt(contentLengthString) : undefined
+
 	const partialFilePath = `${targetFilePath}.${getRandomHexString(16)}.partial`
-	const fileWriteStream = createWriteStream(partialFilePath, { encoding: 'binary', autoClose: true })
+	const fileWriter = new FileWriter(partialFilePath)
 
 	let statusInterval = setInterval(() => {
 		updateStatus()
 	}, statusUpdateInterval)
 
-	response.data.on('data', (chunk: Uint8Array) => {
-		try {
-			const contentLengthString = response.headers['content-length']
-
-			totalBytes = contentLengthString != undefined ? parseInt(contentLengthString) : undefined
-
-			const chunkLength = chunk.length
-
-			fileWriteStream.write(chunk)
-
-			downloadedBytes += chunkLength
-
-			if (downloadStarted == false) {
+	try {
+		for await (const chunk of response.data) {
+			if (downloadStarted === false) {
 				downloadStarted = true
 			}
-		} catch (err) {
-			clearInterval(statusInterval)
 
-			downloadPromise.reject(err)
-		}
-	})
-
-	response.data.on('end', async () => {
-		try {
-			clearInterval(statusInterval)
-			updateStatus()
+			downloadedBytes += chunk.length
 
-			fileWriteStream.end()
+			await fileWriter.write(chunk)
+		}
+	} catch (e) {
+		clearInterval(statusInterval)
 
-			write('\n')
+		await fileWriter.dispose()
 
-			await move(partialFilePath, targetFilePath)
+		throw e
+	}
 
-			downloadPromise.resolve()
-		} catch (err) {
-			clearInterval(statusInterval)
-			downloadPromise.reject(err)
-		}
-	})
+	clearInterval(statusInterval)
+	updateStatus()
 
-	response.data.on('error', async (err: any) => {
-		try {
-			clearInterval(statusInterval)
+	await fileWriter.dispose()
 
-			fileWriteStream.end()
-			await remove(partialFilePath)
-		} finally {
-			downloadPromise.reject(err)
-		}
-	})
+	write('\n')
 
-	return downloadPromise.promise
+	await move(partialFilePath, targetFilePath)
 }

package/src/utilities/FileReader.ts

@@ -0,0 +1,89 @@
+import { open, close, read } from "./FileSystem.js"
+
+export class FileReader {
+	private fileHandle?: number
+	private finished = false
+	private disposed = false
+	private readOffset = 0
+
+	constructor(public readonly filePath: string) {
+	}
+
+	async readChunk(buffer: Uint8Array): Promise<Uint8Array> {
+		if (this.isDisposed) {
+			throw new Error(`FileReader has been disposed`)
+		}
+
+		await this.openIfNeeded()
+
+		let bufferWriteOffset = 0
+
+		while (bufferWriteOffset < buffer.length) {
+			const remainingSizeInBuffer = buffer.length - bufferWriteOffset
+
+			let bytesRead: number
+
+			try {
+				({ bytesRead } = await read(this.fileHandle!, buffer, bufferWriteOffset, remainingSizeInBuffer, this.readOffset))
+			} catch (e) {
+				await this.dispose()
+
+				throw e
+			}
+
+			if (bytesRead === 0) {
+				this.finished = true
+
+				await this.dispose()
+
+				break
+			}
+
+			bufferWriteOffset += bytesRead
+			this.readOffset += bytesRead
+		}
+
+		return buffer.subarray(0, bufferWriteOffset)
+	}
+
+	private async openIfNeeded() {
+		if (this.isDisposed) {
+			throw new Error(`FileWriter has been disposed`)
+		}
+
+		if (this.isOpened) {
+			return
+		}
+
+		this.fileHandle = await open(this.filePath, 'r')
+	}
+
+	async dispose() {
+		if (this.isDisposed) {
+			return
+		}
+
+		if (this.isOpened) {
+			try {
+				await close(this.fileHandle!)
+			} catch (e) {
+			}
+		}
+
+		this.disposed = true
+		this.readOffset = 0
+		this.fileHandle = undefined
+	}
+
+	get isOpened() {
+		return this.fileHandle !== undefined
+	}
+
+	get isDisposed() {
+		return this.disposed
+	}
+
+	get isFinished() {
+		return this.finished
+	}
+}