@zenfs/core 2.7.2 → 2.7.3-newstreams.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backends/fetch.d.ts +2 -0
- package/dist/backends/fetch.js +9 -1
- package/dist/internal/filesystem.d.ts +1 -0
- package/dist/internal/filesystem.js +16 -10
- package/dist/internal/index_fs.d.ts +6 -0
- package/dist/internal/index_fs.js +20 -2
- package/dist/mixins/async.d.ts +5 -6
- package/dist/mixins/async.js +72 -9
- package/dist/node/streams.d.ts +2 -2
- package/dist/node/streams.js +9 -3
- package/dist/node/sync.d.ts +7 -6
- package/dist/node/sync.js +19 -2
- package/dist/vfs/file.js +16 -10
- package/dist/vfs/vnode.js +3 -15
- package/package.json +6 -7
- package/tests/fs/streams.test.ts +60 -0
- package/tests/setup.ts +1 -1
- package/types/README.md +0 -1
- package/types/readable-stream.d.ts +0 -18
package/dist/backends/fetch.d.ts
CHANGED
|
@@ -47,6 +47,8 @@ export declare class FetchFS extends IndexFS {
|
|
|
47
47
|
protected _async(p: Promise<unknown>): void;
|
|
48
48
|
constructor(index: Index, baseUrl: string, requestInit?: RequestInit, remoteWrite?: boolean | undefined);
|
|
49
49
|
protected resize(path: string, inode: Inode, oldSize: number): Promise<void> | void;
|
|
50
|
+
protected _create(path: string, inode: Inode): Promise<void>;
|
|
51
|
+
protected _createSync(path: string, inode: Inode): void;
|
|
50
52
|
touch(path: string, metadata: InodeLike): Promise<void>;
|
|
51
53
|
touchSync(path: string, metadata: InodeLike): void;
|
|
52
54
|
protected remove(path: string): Promise<void>;
|
package/dist/backends/fetch.js
CHANGED
|
@@ -63,6 +63,14 @@ export class FetchFS extends IndexFS {
|
|
|
63
63
|
if (this.remoteWrite)
|
|
64
64
|
return requests.set(url, zeroes, { offset: oldSize, warn }, this.requestInit).catch(parseError);
|
|
65
65
|
}
|
|
66
|
+
async _create(path, inode) {
|
|
67
|
+
await requests
|
|
68
|
+
.set(this.baseUrl + path, new Uint8Array(inode.size), { warn, cacheOnly: !this.remoteWrite, method: 'PUT' }, this.requestInit)
|
|
69
|
+
.catch(parseError);
|
|
70
|
+
}
|
|
71
|
+
_createSync(path, inode) {
|
|
72
|
+
this._async(this._create(path, inode));
|
|
73
|
+
}
|
|
66
74
|
async touch(path, metadata) {
|
|
67
75
|
const inode = this.index.get(path) ?? _throw(withErrno('ENOENT'));
|
|
68
76
|
const oldSize = inode.size;
|
|
@@ -81,7 +89,7 @@ export class FetchFS extends IndexFS {
|
|
|
81
89
|
await requests.remove(this.baseUrl + path, { warn, cacheOnly: !this.remoteWrite }, this.requestInit);
|
|
82
90
|
}
|
|
83
91
|
removeSync(path) {
|
|
84
|
-
this._async(
|
|
92
|
+
this._async(this.remove(path));
|
|
85
93
|
}
|
|
86
94
|
async read(path, buffer, offset = 0, end) {
|
|
87
95
|
const inode = this.index.get(path);
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
2
|
import { withErrno } from 'kerium';
|
|
3
3
|
import { ioctl_default_ops_async, ioctl_default_ops_sync } from './ioctl.js';
|
|
4
|
-
|
|
4
|
+
/** Default bytes per chunk for streamed reads, matching Node's default stream `highWaterMark` */
|
|
5
|
+
const defaultChunkSize = 0x10000;
|
|
5
6
|
/**
|
|
6
7
|
* Provides a consistent and easy to use internal API.
|
|
7
8
|
* Default implementations for `exists` and `existsSync` are included.
|
|
@@ -97,17 +98,22 @@ export class FileSystem {
|
|
|
97
98
|
* @privateRemarks The default implementation of `streamRead` uses "chunked" `read`s
|
|
98
99
|
*/
|
|
99
100
|
streamRead(path, options) {
|
|
101
|
+
const { chunkSize = defaultChunkSize } = options;
|
|
102
|
+
let offset = options.start ?? 0;
|
|
103
|
+
let end = options.end;
|
|
100
104
|
return new ReadableStream({
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
const buffer = new Uint8Array(bytesRead);
|
|
107
|
-
await this.read(path, buffer, offset, offset + bytesRead).catch(controller.error.bind(controller));
|
|
108
|
-
controller.enqueue(buffer);
|
|
105
|
+
pull: async (controller) => {
|
|
106
|
+
end ??= (await this.stat(path)).size;
|
|
107
|
+
if (offset >= end) {
|
|
108
|
+
controller.close();
|
|
109
|
+
return;
|
|
109
110
|
}
|
|
110
|
-
|
|
111
|
+
const buffer = new Uint8Array(Math.min(chunkSize, end - offset));
|
|
112
|
+
await this.read(path, buffer, offset, offset + buffer.byteLength);
|
|
113
|
+
offset += buffer.byteLength;
|
|
114
|
+
controller.enqueue(buffer);
|
|
115
|
+
if (offset >= end)
|
|
116
|
+
controller.close();
|
|
111
117
|
},
|
|
112
118
|
type: 'bytes',
|
|
113
119
|
});
|
|
@@ -5,6 +5,8 @@ import { Inode, type InodeLike } from './inode.js';
|
|
|
5
5
|
* A file system that uses an `Index` for metadata.
|
|
6
6
|
* @category Internals
|
|
7
7
|
* @internal
|
|
8
|
+
*
|
|
9
|
+
* @todo [breaking] replace the three method `create` system with something more intuitive, e.g. requiring subclasses to implement `createFile`, with this just exposing `create` for convenience.
|
|
8
10
|
*/
|
|
9
11
|
export declare abstract class IndexFS extends FileSystem {
|
|
10
12
|
readonly index: Index;
|
|
@@ -27,6 +29,10 @@ export declare abstract class IndexFS extends FileSystem {
|
|
|
27
29
|
unlinkSync(path: string): void;
|
|
28
30
|
rmdir(path: string): Promise<void>;
|
|
29
31
|
rmdirSync(path: string): void;
|
|
32
|
+
/** Create a new file in the underlying backend */
|
|
33
|
+
protected _create?(path: string, inode: Inode): Promise<void>;
|
|
34
|
+
/** Create a new file in the underlying backend */
|
|
35
|
+
protected _createSync?(path: string, inode: Inode): void;
|
|
30
36
|
protected create(path: string, options: CreationOptions): Inode;
|
|
31
37
|
createFile(path: string, options: CreationOptions): Promise<Inode>;
|
|
32
38
|
createFileSync(path: string, options: CreationOptions): Inode;
|
|
@@ -11,6 +11,8 @@ import { Inode } from './inode.js';
|
|
|
11
11
|
* A file system that uses an `Index` for metadata.
|
|
12
12
|
* @category Internals
|
|
13
13
|
* @internal
|
|
14
|
+
*
|
|
15
|
+
* @todo [breaking] replace the three method `create` system with something more intuitive, e.g. requiring subclasses to implement `createFile`, with this just exposing `create` for convenience.
|
|
14
16
|
*/
|
|
15
17
|
export class IndexFS extends FileSystem {
|
|
16
18
|
index;
|
|
@@ -165,10 +167,26 @@ export class IndexFS extends FileSystem {
|
|
|
165
167
|
return inode;
|
|
166
168
|
}
|
|
167
169
|
async createFile(path, options) {
|
|
168
|
-
|
|
170
|
+
const inode = this.create(path, { ...options, mode: options.mode | S_IFREG });
|
|
171
|
+
try {
|
|
172
|
+
await this._create?.(path, inode);
|
|
173
|
+
return inode;
|
|
174
|
+
}
|
|
175
|
+
catch (e) {
|
|
176
|
+
this.index.delete(path);
|
|
177
|
+
throw e;
|
|
178
|
+
}
|
|
169
179
|
}
|
|
170
180
|
createFileSync(path, options) {
|
|
171
|
-
|
|
181
|
+
const inode = this.create(path, { ...options, mode: options.mode | S_IFREG });
|
|
182
|
+
try {
|
|
183
|
+
this._createSync?.(path, inode);
|
|
184
|
+
return inode;
|
|
185
|
+
}
|
|
186
|
+
catch (e) {
|
|
187
|
+
this.index.delete(path);
|
|
188
|
+
throw e;
|
|
189
|
+
}
|
|
172
190
|
}
|
|
173
191
|
async mkdir(path, options) {
|
|
174
192
|
options = { ...options, mode: options.mode | S_IFDIR };
|
package/dist/mixins/async.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Semaphore } from 'kerium/locks';
|
|
1
2
|
import type { FileSystem } from '../internal/filesystem.js';
|
|
2
3
|
import { type _SyncFSKeys, type AsyncFSMethods, type Mixin } from './shared.js';
|
|
3
4
|
/**
|
|
@@ -12,13 +13,11 @@ export type AsyncOperation = {
|
|
|
12
13
|
* @category Internals
|
|
13
14
|
*/
|
|
14
15
|
export interface AsyncMixin extends Pick<FileSystem, Exclude<_SyncFSKeys, 'existsSync'>> {
|
|
15
|
-
/**
|
|
16
|
-
* @internal @protected
|
|
17
|
-
*/
|
|
16
|
+
/** @internal @protected */
|
|
18
17
|
_sync?: FileSystem;
|
|
19
|
-
/**
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
/** Restrict how many files can be copied across at once @internal @protected */
|
|
19
|
+
_crossCopySemaphore?: Semaphore;
|
|
20
|
+
/** @deprecated Use {@link sync | `sync`} instead */
|
|
22
21
|
queueDone(): Promise<void>;
|
|
23
22
|
ready(): Promise<void>;
|
|
24
23
|
sync(): Promise<void>;
|
package/dist/mixins/async.js
CHANGED
|
@@ -1,3 +1,55 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
1
53
|
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
54
|
import { withErrno } from 'kerium';
|
|
3
55
|
import { crit, debug, err } from 'kerium/log';
|
|
@@ -164,19 +216,30 @@ export function Async(FS) {
|
|
|
164
216
|
},
|
|
165
217
|
});
|
|
166
218
|
}
|
|
167
|
-
/**
|
|
168
|
-
|
|
169
|
-
|
|
219
|
+
/** @internal @protected */
|
|
220
|
+
_crossCopySemaphore;
|
|
221
|
+
/** @internal */
|
|
170
222
|
async crossCopy(path) {
|
|
171
223
|
this.checkSync();
|
|
172
224
|
const stats = await this.stat(path);
|
|
173
225
|
if (!isDirectory(stats)) {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
226
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
227
|
+
try {
|
|
228
|
+
const _ = __addDisposableResource(env_1, (await this._crossCopySemaphore?.get()) || { [Symbol.dispose]() { } }, false);
|
|
229
|
+
this._sync.createFileSync(path, stats);
|
|
230
|
+
const buffer = new Uint8Array(stats.size);
|
|
231
|
+
await this.read(path, buffer, 0, stats.size);
|
|
232
|
+
this._sync.writeSync(path, buffer, 0);
|
|
233
|
+
this._sync.touchSync(path, stats);
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
catch (e_1) {
|
|
237
|
+
env_1.error = e_1;
|
|
238
|
+
env_1.hasError = true;
|
|
239
|
+
}
|
|
240
|
+
finally {
|
|
241
|
+
__disposeResources(env_1);
|
|
242
|
+
}
|
|
180
243
|
}
|
|
181
244
|
if (path !== '/') {
|
|
182
245
|
this._sync.mkdirSync(path, stats);
|
package/dist/node/streams.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
/// <reference path="../../types/readable-stream.d.ts" preserve="true" />
|
|
2
1
|
import type { Abortable } from 'node:events';
|
|
3
2
|
import type * as fs from 'node:fs';
|
|
4
3
|
import type { CreateReadStreamOptions, CreateWriteStreamOptions } from 'node:fs/promises';
|
|
5
4
|
import type { Callback } from '../utils.js';
|
|
6
5
|
import type { FileHandle } from './promises.js';
|
|
7
|
-
import { Readable
|
|
6
|
+
import { Readable } from '@zenfs/streams/readable';
|
|
7
|
+
import { Writable } from '@zenfs/streams/writable';
|
|
8
8
|
interface FSImplementation {
|
|
9
9
|
open?: (...args: unknown[]) => unknown;
|
|
10
10
|
close?: (...args: unknown[]) => unknown;
|
package/dist/node/streams.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { eos } from '@zenfs/streams/end-of-stream';
|
|
2
|
+
import { Readable } from '@zenfs/streams/readable';
|
|
3
|
+
import { Writable } from '@zenfs/streams/writable';
|
|
1
4
|
import { Errno, Exception, UV } from 'kerium';
|
|
2
5
|
import { warn } from 'kerium/log';
|
|
3
|
-
import { finished, Readable, Writable } from 'readable-stream';
|
|
4
6
|
/**
|
|
5
7
|
* A ReadStream implementation that wraps an underlying global ReadableStream.
|
|
6
8
|
*/
|
|
@@ -16,7 +18,11 @@ export class ReadStream extends Readable {
|
|
|
16
18
|
.then(handle => {
|
|
17
19
|
this._path = handle['vfs'].path;
|
|
18
20
|
// `end` is inclusive for streams, but the internal range is not
|
|
19
|
-
const internal = handle.readableWebStream({
|
|
21
|
+
const internal = handle.readableWebStream({
|
|
22
|
+
start: opts.start,
|
|
23
|
+
end: opts.end === undefined ? undefined : opts.end + 1,
|
|
24
|
+
chunkSize: opts.highWaterMark,
|
|
25
|
+
});
|
|
20
26
|
this.reader = internal.getReader();
|
|
21
27
|
this.pending = false;
|
|
22
28
|
})
|
|
@@ -46,7 +52,7 @@ export class ReadStream extends Readable {
|
|
|
46
52
|
* Closing before the stream has finished reports `ERR_STREAM_PREMATURE_CLOSE`.
|
|
47
53
|
*/
|
|
48
54
|
close(callback = () => null) {
|
|
49
|
-
|
|
55
|
+
eos(this, error => callback(error ?? null));
|
|
50
56
|
this.destroy();
|
|
51
57
|
}
|
|
52
58
|
get path() {
|
package/dist/node/sync.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { BufferView } from 'node:buffer';
|
|
1
2
|
import type * as fs from 'node:fs';
|
|
2
3
|
import type { V_Context } from '../context.js';
|
|
3
4
|
import type { FileContents } from '../vfs/shared.js';
|
|
@@ -72,14 +73,14 @@ export declare function lopenSync(this: V_Context, path: fs.PathLike, flag: stri
|
|
|
72
73
|
* Synchronously reads the entire contents of a file.
|
|
73
74
|
* @option encoding The string encoding for the file contents. Defaults to `null`.
|
|
74
75
|
* @option flag Defaults to `'r'`.
|
|
76
|
+
* @option buffer A view (or a factory taking the file size) the contents are read into,
|
|
77
|
+
* instead of allocating a new buffer. It must be large enough to hold the entire file.
|
|
75
78
|
* @returns file contents
|
|
76
79
|
*/
|
|
77
|
-
export declare function readFileSync(this: V_Context, path: fs.PathOrFileDescriptor, options
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
export declare function readFileSync(this: V_Context, path: fs.PathOrFileDescriptor, options
|
|
81
|
-
flag?: string;
|
|
82
|
-
}) | BufferEncoding | null): string;
|
|
80
|
+
export declare function readFileSync<T extends NodeJS.ArrayBufferView>(this: V_Context, path: fs.PathOrFileDescriptor, options: fs.ReadFileSyncOptionsWithBuffer<T>): BufferView<T>;
|
|
81
|
+
export declare function readFileSync(this: V_Context, path: fs.PathOrFileDescriptor, options?: fs.ReadFileSyncOptionsWithBufferEncoding | null): NonSharedBuffer;
|
|
82
|
+
export declare function readFileSync(this: V_Context, path: fs.PathOrFileDescriptor, options: fs.ReadFileSyncOptionsWithStringEncoding | BufferEncoding): string;
|
|
83
|
+
export declare function readFileSync(this: V_Context, path: fs.PathOrFileDescriptor, options: fs.ReadFileSyncOptions): string | NonSharedBuffer;
|
|
83
84
|
/**
|
|
84
85
|
* Synchronously writes data to a file, replacing the file if it already exists.
|
|
85
86
|
*
|
package/dist/node/sync.js
CHANGED
|
@@ -175,6 +175,22 @@ openSync;
|
|
|
175
175
|
export function lopenSync(path, flag, mode) {
|
|
176
176
|
return toFD(_sync.open(this, path, { flag, mode, preserveSymlinks: true }));
|
|
177
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* Resolves the `buffer` option of `readFileSync` into a buffer covering the first `size` bytes of the view.
|
|
180
|
+
* @hidden
|
|
181
|
+
*/
|
|
182
|
+
function intoBuffer(into, size) {
|
|
183
|
+
const isFn = typeof into == 'function';
|
|
184
|
+
const view = isFn ? into(size) : into;
|
|
185
|
+
const name = isFn ? 'options.buffer()' : 'options.buffer';
|
|
186
|
+
if (!ArrayBuffer.isView(view))
|
|
187
|
+
throw Object.assign(new TypeError(`The "options.buffer" property must be of type function or an instance of Buffer, TypedArray, or DataView. Received ${typeof view}`), { code: 'ERR_INVALID_ARG_TYPE' });
|
|
188
|
+
if (view.byteLength < size)
|
|
189
|
+
throw Object.assign(new TypeError(`The property '${name}.byteLength' is smaller than the file size of ${size} bytes. Received ${view.byteLength}`), {
|
|
190
|
+
code: 'ERR_INVALID_ARG_VALUE',
|
|
191
|
+
});
|
|
192
|
+
return Buffer.from(view.buffer, view.byteOffset, size);
|
|
193
|
+
}
|
|
178
194
|
export function readFileSync(path, _options = {}) {
|
|
179
195
|
const env_3 = { stack: [], error: void 0, hasError: false };
|
|
180
196
|
try {
|
|
@@ -186,7 +202,8 @@ export function readFileSync(path, _options = {}) {
|
|
|
186
202
|
? fromFD(this, path)
|
|
187
203
|
: _sync.open(this, path.toString(), { flag: options.flag, mode: 0o644, preserveSymlinks: false }), false);
|
|
188
204
|
const { size } = file.stat();
|
|
189
|
-
const
|
|
205
|
+
const into = typeof _options == 'object' && _options !== null ? _options.buffer : undefined;
|
|
206
|
+
const data = into === undefined ? Buffer.alloc(size) : intoBuffer(into, size);
|
|
190
207
|
file.readSync(data, 0, size, 0);
|
|
191
208
|
return options.encoding ? data.toString(options.encoding) : data;
|
|
192
209
|
}
|
|
@@ -584,7 +601,7 @@ export function copyFileSync(source, destination, flags) {
|
|
|
584
601
|
destination = normalizePath.call(this, destination);
|
|
585
602
|
if (flags && flags & constants.COPYFILE_EXCL && existsSync.call(this, destination))
|
|
586
603
|
throw UV('EEXIST', 'copyFile', destination);
|
|
587
|
-
writeFileSync.call(this, destination, readFileSync.call(this, source));
|
|
604
|
+
writeFileSync.call(this, destination, readFileSync.call(this, source, {}));
|
|
588
605
|
emitChange(this, 'rename', destination.toString());
|
|
589
606
|
}
|
|
590
607
|
copyFileSync;
|
package/dist/vfs/file.js
CHANGED
|
@@ -93,8 +93,8 @@ import { _chown, InodeFlags, isBlockDevice, isCharacterDevice } from '../interna
|
|
|
93
93
|
import '../polyfills.js';
|
|
94
94
|
import { validateFD } from '../utils.js';
|
|
95
95
|
import { cacheOf } from './vcache.js';
|
|
96
|
-
/**
|
|
97
|
-
const streamChunkSize =
|
|
96
|
+
/** Default bytes per chunk for streamed reads, matching Node's default stream `highWaterMark` */
|
|
97
|
+
const streamChunkSize = 0x10000;
|
|
98
98
|
/**
|
|
99
99
|
* @internal
|
|
100
100
|
*/
|
|
@@ -608,19 +608,25 @@ let Handle = (() => {
|
|
|
608
608
|
if (this.closed)
|
|
609
609
|
throw UV('EBADF', 'streamRead', this.path);
|
|
610
610
|
const { vnode } = this;
|
|
611
|
+
const { chunkSize = streamChunkSize } = options;
|
|
612
|
+
let offset = options.start ?? 0;
|
|
613
|
+
let end = options.end;
|
|
611
614
|
return new ReadableStream({
|
|
612
|
-
async
|
|
615
|
+
async pull(controller) {
|
|
613
616
|
const env_15 = { stack: [], error: void 0, hasError: false };
|
|
614
617
|
try {
|
|
615
618
|
const _ = __addDisposableResource(env_15, await vnode.lock('ro'), false);
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
await vnode.read(buffer, offset, offset + bytesRead).catch(controller.error.bind(controller));
|
|
621
|
-
controller.enqueue(buffer);
|
|
619
|
+
end ??= vnode.inode.size;
|
|
620
|
+
if (offset >= end) {
|
|
621
|
+
controller.close();
|
|
622
|
+
return;
|
|
622
623
|
}
|
|
623
|
-
|
|
624
|
+
const buffer = new Uint8Array(Math.min(chunkSize, end - offset));
|
|
625
|
+
await vnode.read(buffer, offset, offset + buffer.byteLength);
|
|
626
|
+
offset += buffer.byteLength;
|
|
627
|
+
controller.enqueue(buffer);
|
|
628
|
+
if (offset >= end)
|
|
629
|
+
controller.close();
|
|
624
630
|
}
|
|
625
631
|
catch (e_15) {
|
|
626
632
|
env_15.error = e_15;
|
package/dist/vfs/vnode.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
2
|
import { RwLockable } from 'kerium/locks';
|
|
3
|
-
import { Resource } from 'utilium/cache';
|
|
3
|
+
import { addRange, Resource } from 'utilium/cache';
|
|
4
4
|
import { InodeFlags, isBlockDevice, isCharacterDevice } from '../internal/inode.js';
|
|
5
5
|
/**
|
|
6
6
|
* A VFS-level node, roughly equivalent to Linux's in-memory `struct inode`.
|
|
@@ -57,17 +57,7 @@ export class VNode extends RwLockable {
|
|
|
57
57
|
}
|
|
58
58
|
/** Add a range to `dirtyRanges`, merging overlapping and adjacent ranges */
|
|
59
59
|
markDirty(start, end) {
|
|
60
|
-
this.dirtyRanges
|
|
61
|
-
this.dirtyRanges.sort((a, b) => a.start - b.start);
|
|
62
|
-
const merged = [];
|
|
63
|
-
for (const range of this.dirtyRanges) {
|
|
64
|
-
const last = merged.at(-1);
|
|
65
|
-
if (last && range.start <= last.end)
|
|
66
|
-
last.end = Math.max(last.end, range.end);
|
|
67
|
-
else
|
|
68
|
-
merged.push(range);
|
|
69
|
-
}
|
|
70
|
-
this.dirtyRanges = merged;
|
|
60
|
+
addRange(this.dirtyRanges, start, end);
|
|
71
61
|
}
|
|
72
62
|
/**
|
|
73
63
|
* Copy `[start, end)` from the cache into `buffer`.
|
|
@@ -75,12 +65,10 @@ export class VNode extends RwLockable {
|
|
|
75
65
|
*/
|
|
76
66
|
copyFromCache(buffer, start, end) {
|
|
77
67
|
let pos = start;
|
|
78
|
-
for (const region of this.data.
|
|
68
|
+
for (const region of this.data.regionsIn(start, end)) {
|
|
79
69
|
const regionEnd = region.offset + region.data.byteLength;
|
|
80
70
|
if (regionEnd <= pos)
|
|
81
71
|
continue;
|
|
82
|
-
if (region.offset >= end)
|
|
83
|
-
break;
|
|
84
72
|
if (region.offset > pos) {
|
|
85
73
|
buffer.fill(0, pos - start, region.offset - start);
|
|
86
74
|
pos = region.offset;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zenfs/core",
|
|
3
|
-
"version": "2.7.
|
|
3
|
+
"version": "2.7.3-newstreams.0",
|
|
4
4
|
"description": "A filesystem, anywhere",
|
|
5
5
|
"funding": {
|
|
6
6
|
"type": "individual",
|
|
@@ -69,19 +69,18 @@
|
|
|
69
69
|
"lint": "eslint src tests",
|
|
70
70
|
"test": "npx -s zenfs-test --clean; npx -s zenfs-test -abcpn; tests/fetch/run.sh; npx -s zenfs-test --report-only",
|
|
71
71
|
"test:native": "npx -s zenfs-test tests/native-setup.ts --verbose",
|
|
72
|
-
"build": "tsc
|
|
73
|
-
"
|
|
74
|
-
"dev": "tsc -p tsconfig.json --watch",
|
|
72
|
+
"build": "tsc --build",
|
|
73
|
+
"dev": "tsc --build --watch",
|
|
75
74
|
"prepublishOnly": "npm run build"
|
|
76
75
|
},
|
|
77
76
|
"dependencies": {
|
|
78
77
|
"@types/node": "^26.0.0",
|
|
78
|
+
"@zenfs/streams": "^0.0.1",
|
|
79
79
|
"buffer": "^6.0.3",
|
|
80
80
|
"eventemitter3": "^5.0.1",
|
|
81
|
-
"kerium": "^1.
|
|
81
|
+
"kerium": "^1.5.0",
|
|
82
82
|
"memium": "^1.0.0",
|
|
83
|
-
"
|
|
84
|
-
"utilium": "^3.0.0"
|
|
83
|
+
"utilium": "^3.7.0"
|
|
85
84
|
},
|
|
86
85
|
"devDependencies": {
|
|
87
86
|
"@eslint/js": "^10.0.1",
|
package/tests/fs/streams.test.ts
CHANGED
|
@@ -141,4 +141,64 @@ suite('Streams', config('streams'), () => {
|
|
|
141
141
|
// Closing invalidates the descriptor, so it fails range validation
|
|
142
142
|
assert.throws(() => fileHandle.createWriteStream(), { code: 'ERR_OUT_OF_RANGE' });
|
|
143
143
|
});
|
|
144
|
+
|
|
145
|
+
// A file large enough to span several chunks, so the read path has to stitch
|
|
146
|
+
// them back together and the vnode cache has to grow to hold them.
|
|
147
|
+
const large = Buffer.alloc(200 * 1024);
|
|
148
|
+
for (let i = 0; i < large.length; i++) large[i] = (i * 31 + 7) & 0xff;
|
|
149
|
+
|
|
150
|
+
test('createReadStream reads a multi-chunk file byte for byte', async () => {
|
|
151
|
+
await fs.promises.writeFile('large.bin', large);
|
|
152
|
+
|
|
153
|
+
const chunks: Uint8Array[] = [];
|
|
154
|
+
for await (const chunk of fs.createReadStream('large.bin')) chunks.push(chunk as Uint8Array);
|
|
155
|
+
|
|
156
|
+
assert.deepEqual(Buffer.concat(chunks), large);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test('createReadStream honors highWaterMark as the chunk size', async () => {
|
|
160
|
+
await fs.promises.writeFile('large.bin', large);
|
|
161
|
+
|
|
162
|
+
const highWaterMark = 64 * 1024;
|
|
163
|
+
const sizes: number[] = [];
|
|
164
|
+
for await (const chunk of fs.createReadStream('large.bin', { highWaterMark })) sizes.push((chunk as Uint8Array).byteLength);
|
|
165
|
+
|
|
166
|
+
// Every chunk but the last is a full highWaterMark, and they sum to the file
|
|
167
|
+
assert.deepEqual(sizes.slice(0, -1), Array(sizes.length - 1).fill(highWaterMark));
|
|
168
|
+
assert.equal(
|
|
169
|
+
sizes.reduce((a, b) => a + b, 0),
|
|
170
|
+
large.length
|
|
171
|
+
);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test('createReadStream with start and end spanning chunks', async () => {
|
|
175
|
+
await fs.promises.writeFile('large.bin', large);
|
|
176
|
+
|
|
177
|
+
// `end` is inclusive, and this range deliberately starts and stops mid-chunk
|
|
178
|
+
const start = 1000;
|
|
179
|
+
const end = 150_000;
|
|
180
|
+
const chunks: Uint8Array[] = [];
|
|
181
|
+
for await (const chunk of fs.createReadStream('large.bin', { start, end, highWaterMark: 8192 })) chunks.push(chunk as Uint8Array);
|
|
182
|
+
|
|
183
|
+
assert.deepEqual(Buffer.concat(chunks), large.subarray(start, end + 1));
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test('a read stream does not read past the end of the file', async () => {
|
|
187
|
+
await fs.promises.writeFile('small.bin', large.subarray(0, 10));
|
|
188
|
+
|
|
189
|
+
const chunks: Uint8Array[] = [];
|
|
190
|
+
for await (const chunk of fs.createReadStream('small.bin', { highWaterMark: 64 * 1024 })) chunks.push(chunk as Uint8Array);
|
|
191
|
+
|
|
192
|
+
assert.equal(chunks.length, 1);
|
|
193
|
+
assert.deepEqual(chunks[0], large.subarray(0, 10));
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test('a read stream over an empty file ends without a chunk', async () => {
|
|
197
|
+
await fs.promises.writeFile('empty.bin', '');
|
|
198
|
+
|
|
199
|
+
const chunks: unknown[] = [];
|
|
200
|
+
for await (const chunk of fs.createReadStream('empty.bin')) chunks.push(chunk);
|
|
201
|
+
|
|
202
|
+
assert.deepEqual(chunks, []);
|
|
203
|
+
});
|
|
144
204
|
});
|
package/tests/setup.ts
CHANGED
|
@@ -8,7 +8,7 @@ export const data = join(import.meta.dirname, 'data');
|
|
|
8
8
|
|
|
9
9
|
export const tmp = join(import.meta.dirname, 'tmp');
|
|
10
10
|
|
|
11
|
-
if (!existsSync(tmp)) mkdirSync(tmp);
|
|
11
|
+
if (!existsSync(tmp)) mkdirSync(tmp, { recursive: true });
|
|
12
12
|
|
|
13
13
|
export async function copyAsync(_path: string, fs: NodeFS = _fs): Promise<void> {
|
|
14
14
|
const path = relative(data, _path) || '/';
|
package/types/README.md
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
This directory contains declaration files. Due to numerous TypeScript shenanigans, placing them into `src` is a really bad idea. Instead, we put them here.
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
@types/readable-stream has *many* problems:
|
|
3
|
-
https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71923
|
|
4
|
-
https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71307
|
|
5
|
-
https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71083
|
|
6
|
-
https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/66049
|
|
7
|
-
... and many more
|
|
8
|
-
|
|
9
|
-
This allows us to bypass those problems.
|
|
10
|
-
|
|
11
|
-
Warning: Do not install @types/readable-stream alongside this package!
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
/// <reference types="node" preserve="true" />
|
|
15
|
-
declare module 'readable-stream' {
|
|
16
|
-
import { finished, Readable, Writable } from 'node:stream';
|
|
17
|
-
export { finished, Readable, Writable };
|
|
18
|
-
}
|