@zenfs/core 2.7.3-newstreams.0 → 2.7.4

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@zenfs/core",
-	"version": "2.7.3-newstreams.0",
+	"version": "2.7.4",
 	"description": "A filesystem, anywhere",
 	"funding": {
 		"type": "individual",
@@ -69,13 +69,14 @@
 		"lint": "eslint src tests",
 		"test": "npx -s zenfs-test --clean; npx -s zenfs-test -abcpn; tests/fetch/run.sh; npx -s zenfs-test --report-only",
 		"test:native": "npx -s zenfs-test tests/native-setup.ts --verbose",
+		"test:perf": "zbench",
 		"build": "tsc --build",
 		"dev": "tsc --build --watch",
 		"prepublishOnly": "npm run build"
 	},
 	"dependencies": {
 		"@types/node": "^26.0.0",
-		"@zenfs/streams": "^0.0.1",
+		"@zenfs/streams": "^1.0.0",
 		"buffer": "^6.0.3",
 		"eventemitter3": "^5.0.1",
 		"kerium": "^1.5.0",
@@ -91,7 +92,8 @@
 		"prettier": "^3.2.5",
 		"typedoc": "^0.28.18",
 		"typescript": "^6.0.0",
-		"typescript-eslint": "^8.58.0"
+		"typescript-eslint": "^8.58.0",
+		"zbench-js": "^0.1.0"
 	},
 	"prettier": {
 		"singleQuote": true,

package/tests/perf/append.ts

@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Sequential appends into one region. Every write extends the same region, so a region that
+// allocates exactly what it needs copies the whole file on each append.
+import type { FileHandle } from '@zenfs/core/promises';
+import { fs, mount, piece } from './common.ts';
+
+interface Config {
+	writes: number;
+}
+
+interface State {
+	data: Buffer;
+	handle?: FileHandle;
+}
+
+export async function setup(): Promise<State> {
+	await mount('perf-append');
+	return { data: Buffer.alloc(piece, 0x11) };
+}
+
+export async function before(config: Config, state: State): Promise<void> {
+	state.handle = await fs.promises.open('/a', 'w');
+}
+
+export async function test(config: Config, state: State) {
+	for (let i = 0; i < config.writes; i++) await state.handle!.write(state.data, 0, state.data.length, i * piece);
+
+	return { writes: config.writes, bytes: config.writes * piece };
+}
+
+export async function after(config: Config, state: State): Promise<void> {
+	await state.handle!.close();
+	await fs.promises.rm('/a');
+}

package/tests/perf/common.ts

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import { configureSingle, fs, InMemory } from '@zenfs/core';
+
+export { fs };
+
+export const KB = 1024;
+export const MB = 1024 * 1024;
+
+/**
+ * A piece and the stride between pieces, chosen so consecutive writes never merge into one region:
+ * the 12 KB hole between them is wider than `regionGapThreshold`.
+ */
+export const piece = 4 * KB;
+export const stride = 16 * KB;
+
+/** A fresh in-memory filesystem, so no configuration inherits another's cached vnodes. */
+export async function mount(name: string): Promise<void> {
+	await configureSingle({ backend: InMemory, name });
+}
+
+/** Split a buffer the way a stream writer feeds it. */
+export function chunks(buffer: Buffer, size: number): Buffer[] {
+	const out: Buffer[] = [];
+	for (let i = 0; i < buffer.length; i += size) out.push(buffer.subarray(i, Math.min(i + size, buffer.length)));
+	return out;
+}
+
+/** Give the event loop a moment, so work queued by the previous phase never lands in the timed one. */
+export function settle(): Promise<void> {
+	return new Promise(resolve => setTimeout(resolve, 50));
+}

package/tests/perf/config.json

@@ -0,0 +1,84 @@
+{
+	"flags": {
+		"prep": ["writeFile", "stream"]
+	},
+	"tests": [
+		{
+			"path": "stream-read.ts",
+			"name": "Read with createReadStream",
+			"flags": ["prep"],
+			"measure": {
+				"bytes": { "throughput": "s", "aggregate_throughput": "s", "unit": "MB" },
+				"files": { "cost": true, "unit": "file" }
+			},
+			"configurations": [
+				{ "name": "4K x2000", "value": { "size": 4096, "count": 2000, "hwm": 65536 } },
+				{ "name": "256K x200", "value": { "size": 262144, "count": 200, "hwm": 65536 } },
+				{ "name": "8M x8", "cpu": 1, "mem": 1, "value": { "size": 8388608, "count": 8, "hwm": 65536 } },
+				{
+					"name": "8M x8, hwm 1M",
+					"cpu": 1,
+					"mem": 1,
+					"value": { "size": 8388608, "count": 8, "hwm": 1048576 }
+				}
+			]
+		},
+		{
+			"path": "stream-write.ts",
+			"name": "Write with createWriteStream",
+			"measure": {
+				"bytes": { "throughput": "s", "aggregate_throughput": "s", "unit": "MB" },
+				"files": { "cost": true, "unit": "file" }
+			},
+			"configurations": [
+				{ "name": "4K x2000", "value": { "size": 4096, "count": 2000, "hwm": 65536 } },
+				{ "name": "256K x200", "value": { "size": 262144, "count": 200, "hwm": 65536 } },
+				{ "name": "8M x8", "cpu": 1, "mem": 1, "value": { "size": 8388608, "count": 8, "hwm": 65536 } },
+				{
+					"name": "8M x8, hwm 1M",
+					"cpu": 1,
+					"mem": 1,
+					"value": { "size": 8388608, "count": 8, "hwm": 1048576 }
+				}
+			]
+		},
+		{
+			"path": "fragmented-read.ts",
+			"name": "Read across many cached regions",
+			"measure": {
+				"reads": { "cost": true, "aggregate_cost": true, "unit": "read" }
+			},
+			"configurations": [
+				{ "name": "128 regions", "value": { "regions": 128, "reads": 4000 } },
+				{ "name": "512 regions", "value": { "regions": 512, "reads": 4000 } },
+				{ "name": "2048 regions", "cpu": 1, "value": { "regions": 2048, "reads": 4000 } },
+				{ "name": "8192 regions", "cpu": 2, "mem": 1, "value": { "regions": 8192, "reads": 4000 } }
+			]
+		},
+		{
+			"path": "fragmented-write.ts",
+			"name": "Scattered writes into one file",
+			"measure": {
+				"writes": { "cost": true, "aggregate_cost": true, "unit": "write" }
+			},
+			"configurations": [
+				{ "name": "512 writes", "value": { "writes": 512 } },
+				{ "name": "2048 writes", "cpu": 1, "value": { "writes": 2048 } },
+				{ "name": "8192 writes", "cpu": 2, "mem": 1, "value": { "writes": 8192 } }
+			]
+		},
+		{
+			"path": "append.ts",
+			"name": "Append 4 KB at a time",
+			"measure": {
+				"bytes": { "throughput": "s", "unit": "MB" },
+				"writes": { "cost": true, "aggregate_cost": true, "unit": "write" }
+			},
+			"configurations": [
+				{ "name": "500 appends", "value": { "writes": 500 } },
+				{ "name": "2000 appends", "value": { "writes": 2000 } },
+				{ "name": "8000 appends", "cpu": 1, "value": { "writes": 8000 } }
+			]
+		}
+	]
+}

package/tests/perf/fragmented-read.ts

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Reads spread across a file with many disjoint cached regions.
+
+import type { FileHandle } from '@zenfs/core/promises';
+import { fs, mount, piece, stride } from './common.ts';
+
+interface Config {
+	/** Cached regions to build, one per scattered write */
+	regions: number;
+	reads: number;
+}
+
+interface State {
+	handle: FileHandle;
+	target: Buffer;
+}
+
+export async function setup(config: Config): Promise<State> {
+	await mount('perf-fragmented-read');
+
+	await fs.promises.writeFile('/f', Buffer.alloc(config.regions * stride));
+	const handle = await fs.promises.open('/f', 'r+');
+
+	const data = Buffer.alloc(piece, 0xcd);
+	for (let i = 0; i < config.regions; i++) await handle.write(data, 0, data.length, i * stride);
+
+	return { handle, target: Buffer.allocUnsafe(piece) };
+}
+
+export async function test(config: Config, state: State) {
+	// Backwards, so a scan that starts at region zero pays its worst case on every read
+	for (let i = 0; i < config.reads; i++)
+		await state.handle.read(state.target, 0, state.target.length, (config.regions - 1 - (i % config.regions)) * stride);
+
+	return { reads: config.reads, bytes: config.reads * piece };
+}
+
+export async function teardown(config: Config, state: State): Promise<void> {
+	await state.handle.close();
+	await fs.promises.rm('/f');
+}

package/tests/perf/fragmented-write.ts

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Writes that each land in their own region, so the region list grows by one per write.
+// Anything that walks the whole list — or re-sorts the dirty ranges — per write shows up as
+// quadratic here.
+import type { FileHandle } from '@zenfs/core/promises';
+import { fs, mount, piece, stride } from './common.ts';
+
+interface Config {
+	writes: number;
+}
+
+interface State {
+	data: Buffer;
+	handle?: FileHandle;
+}
+
+export async function setup(): Promise<State> {
+	await mount('perf-fragmented-write');
+	return { data: Buffer.alloc(piece, 0xcd) };
+}
+
+/** A fresh file each iteration: the point is the cost of building the region list, not of reusing it. */
+export async function before(config: Config, state: State): Promise<void> {
+	await fs.promises.writeFile('/f', Buffer.alloc(config.writes * stride));
+	state.handle = await fs.promises.open('/f', 'r+');
+}
+
+export async function test(config: Config, state: State) {
+	for (let i = 0; i < config.writes; i++) await state.handle!.write(state.data, 0, state.data.length, i * stride);
+
+	return { writes: config.writes, bytes: config.writes * piece };
+}
+
+export async function after(config: Config, state: State): Promise<void> {
+	await state.handle!.close();
+	await fs.promises.rm('/f');
+}

package/tests/perf/stream-read.ts

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Reading through `createReadStream`: the VFS cache lookup, `copyFromCache`, and the stream's own
+// chunking, with the files written outside the timed window.
+import { pipeline } from 'node:stream/promises';
+import { chunks, fs, mount, settle } from './common.ts';
+
+interface Config {
+	size: number;
+	count: number;
+	/** Stream highWaterMark, which is also the read chunk size */
+	hwm: number;
+	prep: 'writeFile' | 'stream';
+}
+
+interface State {
+	buffer: Buffer;
+	parts: Buffer[];
+}
+
+export async function setup(config: Config): Promise<State> {
+	await mount('perf-stream-read');
+	const buffer = Buffer.alloc(config.size, 0xab);
+	return { buffer, parts: chunks(buffer, config.hwm) };
+}
+
+export async function before(config: Config, state: State): Promise<void> {
+	for (let i = 0; i < config.count; i++) {
+		if (config.prep == 'stream') await pipeline(state.parts, fs.createWriteStream(`/w${i}`, { highWaterMark: config.hwm }));
+		else await fs.promises.writeFile(`/w${i}`, state.buffer);
+	}
+	await settle();
+}
+
+export async function test(config: Config) {
+	let bytes = 0;
+
+	for (let i = 0; i < config.count; i++) {
+		for await (const chunk of fs.createReadStream(`/w${i}`, { highWaterMark: config.hwm })) bytes += (chunk as Buffer).length;
+	}
+
+	if (bytes != config.size * config.count) throw new Error(`short read: ${bytes} of ${config.size * config.count}`);
+
+	return { bytes, files: config.count };
+}
+
+export async function after(config: Config): Promise<void> {
+	for (let i = 0; i < config.count; i++) await fs.promises.rm(`/w${i}`);
+}

package/tests/perf/stream-write.ts

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Writing through `createWriteStream`: how the cache grows as a file is appended to one chunk at a
+// time. A region that reallocates exactly on every append makes this quadratic in the file's size.
+import { pipeline } from 'node:stream/promises';
+import { chunks, fs, mount, settle } from './common.ts';
+
+interface Config {
+	size: number;
+	count: number;
+	hwm: number;
+}
+
+interface State {
+	parts: Buffer[];
+}
+
+export async function setup(config: Config): Promise<State> {
+	await mount('perf-stream-write');
+	return { parts: chunks(Buffer.alloc(config.size, 0xab), config.hwm) };
+}
+
+export async function test(config: Config, state: State) {
+	for (let i = 0; i < config.count; i++) await pipeline(state.parts, fs.createWriteStream(`/w${i}`, { highWaterMark: config.hwm }));
+
+	return { bytes: config.size * config.count, files: config.count };
+}
+
+export async function after(config: Config): Promise<void> {
+	for (let i = 0; i < config.count; i++) await fs.promises.rm(`/w${i}`);
+	await settle();
+}