@zenfs/core 2.7.3-newstreams.0 → 2.7.3

package/dist/internal/filesystem.d.ts

@@ -117,7 +117,6 @@ export interface StreamOptions {
     start?: number;
     end?: number;
     autoClose?: boolean;
-    chunkSize?: number;
 }
 /**
  * Provides a consistent and easy to use internal API.

package/dist/internal/filesystem.js

@@ -1,8 +1,7 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 import { withErrno } from 'kerium';
 import { ioctl_default_ops_async, ioctl_default_ops_sync } from './ioctl.js';
-/** Default bytes per chunk for streamed reads, matching Node's default stream `highWaterMark` */
-const defaultChunkSize = 0x10000;
+const _chunkSize = 0x1000;
 /**
  * Provides a consistent and easy to use internal API.
  * Default implementations for `exists` and `existsSync` are included.
@@ -98,22 +97,17 @@ export class FileSystem {
      * @privateRemarks The default implementation of `streamRead` uses "chunked" `read`s
      */
     streamRead(path, options) {
-        const { chunkSize = defaultChunkSize } = options;
-        let offset = options.start ?? 0;
-        let end = options.end;
         return new ReadableStream({
-            pull: async (controller) => {
-                end ??= (await this.stat(path)).size;
-                if (offset >= end) {
-                    controller.close();
-                    return;
+            start: async (controller) => {
+                const { size } = await this.stat(path);
+                const { start = 0, end = size } = options;
+                for (let offset = start; offset < end; offset += _chunkSize) {
+                    const bytesRead = offset + _chunkSize > end ? end - offset : _chunkSize;
+                    const buffer = new Uint8Array(bytesRead);
+                    await this.read(path, buffer, offset, offset + bytesRead).catch(controller.error.bind(controller));
+                    controller.enqueue(buffer);
                 }
-                const buffer = new Uint8Array(Math.min(chunkSize, end - offset));
-                await this.read(path, buffer, offset, offset + buffer.byteLength);
-                offset += buffer.byteLength;
-                controller.enqueue(buffer);
-                if (offset >= end)
-                    controller.close();
+                controller.close();
             },
             type: 'bytes',
         });

package/dist/node/streams.d.ts

@@ -1,10 +1,10 @@
+/// <reference path="../../types/readable-stream.d.ts" preserve="true" />
 import type { Abortable } from 'node:events';
 import type * as fs from 'node:fs';
 import type { CreateReadStreamOptions, CreateWriteStreamOptions } from 'node:fs/promises';
 import type { Callback } from '../utils.js';
 import type { FileHandle } from './promises.js';
-import { Readable } from '@zenfs/streams/readable';
-import { Writable } from '@zenfs/streams/writable';
+import { Readable, Writable } from 'readable-stream';
 interface FSImplementation {
     open?: (...args: unknown[]) => unknown;
     close?: (...args: unknown[]) => unknown;

package/dist/node/streams.js

@@ -1,8 +1,6 @@
-import { eos } from '@zenfs/streams/end-of-stream';
-import { Readable } from '@zenfs/streams/readable';
-import { Writable } from '@zenfs/streams/writable';
 import { Errno, Exception, UV } from 'kerium';
 import { warn } from 'kerium/log';
+import { finished, Readable, Writable } from 'readable-stream';
 /**
  * A ReadStream implementation that wraps an underlying global ReadableStream.
  */
@@ -18,11 +16,7 @@ export class ReadStream extends Readable {
             .then(handle => {
             this._path = handle['vfs'].path;
             // `end` is inclusive for streams, but the internal range is not
-            const internal = handle.readableWebStream({
-                start: opts.start,
-                end: opts.end === undefined ? undefined : opts.end + 1,
-                chunkSize: opts.highWaterMark,
-            });
+            const internal = handle.readableWebStream({ start: opts.start, end: opts.end === undefined ? undefined : opts.end + 1 });
             this.reader = internal.getReader();
             this.pending = false;
         })
@@ -52,7 +46,7 @@ export class ReadStream extends Readable {
      * Closing before the stream has finished reports `ERR_STREAM_PREMATURE_CLOSE`.
      */
     close(callback = () => null) {
-        eos(this, error => callback(error ?? null));
+        finished(this, error => callback(error ?? null));
         this.destroy();
     }
     get path() {

package/dist/vfs/file.js

@@ -93,8 +93,8 @@ import { _chown, InodeFlags, isBlockDevice, isCharacterDevice } from '../interna
 import '../polyfills.js';
 import { validateFD } from '../utils.js';
 import { cacheOf } from './vcache.js';
-/** Default bytes per chunk for streamed reads, matching Node's default stream `highWaterMark` */
-const streamChunkSize = 0x10000;
+/** The chunk size used when streaming reads and writes */
+const streamChunkSize = 0x1000;
 /**
  * @internal
  */
@@ -608,25 +608,19 @@ let Handle = (() => {
             if (this.closed)
                 throw UV('EBADF', 'streamRead', this.path);
             const { vnode } = this;
-            const { chunkSize = streamChunkSize } = options;
-            let offset = options.start ?? 0;
-            let end = options.end;
             return new ReadableStream({
-                async pull(controller) {
+                async start(controller) {
                     const env_15 = { stack: [], error: void 0, hasError: false };
                     try {
                         const _ = __addDisposableResource(env_15, await vnode.lock('ro'), false);
-                        end ??= vnode.inode.size;
-                        if (offset >= end) {
-                            controller.close();
-                            return;
+                        const { start = 0, end = vnode.inode.size } = options;
+                        for (let offset = start; offset < end; offset += streamChunkSize) {
+                            const bytesRead = offset + streamChunkSize > end ? end - offset : streamChunkSize;
+                            const buffer = new Uint8Array(bytesRead);
+                            await vnode.read(buffer, offset, offset + bytesRead).catch(controller.error.bind(controller));
+                            controller.enqueue(buffer);
                         }
-                        const buffer = new Uint8Array(Math.min(chunkSize, end - offset));
-                        await vnode.read(buffer, offset, offset + buffer.byteLength);
-                        offset += buffer.byteLength;
-                        controller.enqueue(buffer);
-                        if (offset >= end)
-                            controller.close();
+                        controller.close();
                     }
                     catch (e_15) {
                         env_15.error = e_15;

package/dist/vfs/vnode.js

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 import { RwLockable } from 'kerium/locks';
-import { addRange, Resource } from 'utilium/cache';
+import { Resource } from 'utilium/cache';
 import { InodeFlags, isBlockDevice, isCharacterDevice } from '../internal/inode.js';
 /**
  * A VFS-level node, roughly equivalent to Linux's in-memory `struct inode`.
@@ -57,7 +57,17 @@ export class VNode extends RwLockable {
     }
     /** Add a range to `dirtyRanges`, merging overlapping and adjacent ranges */
     markDirty(start, end) {
-        addRange(this.dirtyRanges, start, end);
+        this.dirtyRanges.push({ start, end });
+        this.dirtyRanges.sort((a, b) => a.start - b.start);
+        const merged = [];
+        for (const range of this.dirtyRanges) {
+            const last = merged.at(-1);
+            if (last && range.start <= last.end)
+                last.end = Math.max(last.end, range.end);
+            else
+                merged.push(range);
+        }
+        this.dirtyRanges = merged;
     }
     /**
      * Copy `[start, end)` from the cache into `buffer`.
@@ -65,10 +75,12 @@ export class VNode extends RwLockable {
      */
     copyFromCache(buffer, start, end) {
         let pos = start;
-        for (const region of this.data.regionsIn(start, end)) {
+        for (const region of this.data.regions) {
             const regionEnd = region.offset + region.data.byteLength;
             if (regionEnd <= pos)
                 continue;
+            if (region.offset >= end)
+                break;
             if (region.offset > pos) {
                 buffer.fill(0, pos - start, region.offset - start);
                 pos = region.offset;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@zenfs/core",
-	"version": "2.7.3-newstreams.0",
+	"version": "2.7.3",
 	"description": "A filesystem, anywhere",
 	"funding": {
 		"type": "individual",
@@ -75,12 +75,12 @@
 	},
 	"dependencies": {
 		"@types/node": "^26.0.0",
-		"@zenfs/streams": "^0.0.1",
 		"buffer": "^6.0.3",
 		"eventemitter3": "^5.0.1",
 		"kerium": "^1.5.0",
 		"memium": "^1.0.0",
-		"utilium": "^3.7.0"
+		"readable-stream": "^4.5.2",
+		"utilium": "^3.0.0"
 	},
 	"devDependencies": {
 		"@eslint/js": "^10.0.1",

package/tests/fs/streams.test.ts

@@ -141,64 +141,4 @@ suite('Streams', config('streams'), () => {
 		// Closing invalidates the descriptor, so it fails range validation
 		assert.throws(() => fileHandle.createWriteStream(), { code: 'ERR_OUT_OF_RANGE' });
 	});
-
-	// A file large enough to span several chunks, so the read path has to stitch
-	// them back together and the vnode cache has to grow to hold them.
-	const large = Buffer.alloc(200 * 1024);
-	for (let i = 0; i < large.length; i++) large[i] = (i * 31 + 7) & 0xff;
-
-	test('createReadStream reads a multi-chunk file byte for byte', async () => {
-		await fs.promises.writeFile('large.bin', large);
-
-		const chunks: Uint8Array[] = [];
-		for await (const chunk of fs.createReadStream('large.bin')) chunks.push(chunk as Uint8Array);
-
-		assert.deepEqual(Buffer.concat(chunks), large);
-	});
-
-	test('createReadStream honors highWaterMark as the chunk size', async () => {
-		await fs.promises.writeFile('large.bin', large);
-
-		const highWaterMark = 64 * 1024;
-		const sizes: number[] = [];
-		for await (const chunk of fs.createReadStream('large.bin', { highWaterMark })) sizes.push((chunk as Uint8Array).byteLength);
-
-		// Every chunk but the last is a full highWaterMark, and they sum to the file
-		assert.deepEqual(sizes.slice(0, -1), Array(sizes.length - 1).fill(highWaterMark));
-		assert.equal(
-			sizes.reduce((a, b) => a + b, 0),
-			large.length
-		);
-	});
-
-	test('createReadStream with start and end spanning chunks', async () => {
-		await fs.promises.writeFile('large.bin', large);
-
-		// `end` is inclusive, and this range deliberately starts and stops mid-chunk
-		const start = 1000;
-		const end = 150_000;
-		const chunks: Uint8Array[] = [];
-		for await (const chunk of fs.createReadStream('large.bin', { start, end, highWaterMark: 8192 })) chunks.push(chunk as Uint8Array);
-
-		assert.deepEqual(Buffer.concat(chunks), large.subarray(start, end + 1));
-	});
-
-	test('a read stream does not read past the end of the file', async () => {
-		await fs.promises.writeFile('small.bin', large.subarray(0, 10));
-
-		const chunks: Uint8Array[] = [];
-		for await (const chunk of fs.createReadStream('small.bin', { highWaterMark: 64 * 1024 })) chunks.push(chunk as Uint8Array);
-
-		assert.equal(chunks.length, 1);
-		assert.deepEqual(chunks[0], large.subarray(0, 10));
-	});
-
-	test('a read stream over an empty file ends without a chunk', async () => {
-		await fs.promises.writeFile('empty.bin', '');
-
-		const chunks: unknown[] = [];
-		for await (const chunk of fs.createReadStream('empty.bin')) chunks.push(chunk);
-
-		assert.deepEqual(chunks, []);
-	});
 });

package/tests/setup.ts

@@ -8,7 +8,7 @@ export const data = join(import.meta.dirname, 'data');
 
 export const tmp = join(import.meta.dirname, 'tmp');
 
-if (!existsSync(tmp)) mkdirSync(tmp, { recursive: true });
+if (!existsSync(tmp)) mkdirSync(tmp);
 
 export async function copyAsync(_path: string, fs: NodeFS = _fs): Promise<void> {
 	const path = relative(data, _path) || '/';

package/types/README.md

@@ -0,0 +1 @@
+This directory contains declaration files. Due to numerous TypeScript shenanigans, placing them into `src` is a really bad idea. Instead, we put them here.

package/types/readable-stream.d.ts

@@ -0,0 +1,18 @@
+/*
+@types/readable-stream has *many* problems:
+https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71923
+https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71307
+https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71083
+https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/66049
+... and many more
+
+This allows us to bypass those problems.
+
+Warning: Do not install @types/readable-stream alongside this package!
+*/
+
+/// <reference types="node" preserve="true" />
+declare module 'readable-stream' {
+	import { finished, Readable, Writable } from 'node:stream';
+	export { finished, Readable, Writable };
+}