@zenfs/core 2.7.3 → 2.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/internal/filesystem.d.ts +1 -0
- package/dist/internal/filesystem.js +16 -10
- package/dist/node/streams.d.ts +2 -2
- package/dist/node/streams.js +9 -3
- package/dist/vfs/file.js +16 -10
- package/dist/vfs/vnode.js +3 -15
- package/package.json +6 -4
- package/tests/fs/streams.test.ts +60 -0
- package/tests/perf/append.ts +34 -0
- package/tests/perf/common.ts +31 -0
- package/tests/perf/config.json +84 -0
- package/tests/perf/fragmented-read.ts +41 -0
- package/tests/perf/fragmented-write.ts +37 -0
- package/tests/perf/stream-read.ts +48 -0
- package/tests/perf/stream-write.ts +31 -0
- package/tests/setup.ts +1 -1
- package/types/README.md +0 -1
- package/types/readable-stream.d.ts +0 -18
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
2
|
import { withErrno } from 'kerium';
|
|
3
3
|
import { ioctl_default_ops_async, ioctl_default_ops_sync } from './ioctl.js';
|
|
4
|
-
|
|
4
|
+
/** Default bytes per chunk for streamed reads, matching Node's default stream `highWaterMark` */
|
|
5
|
+
const defaultChunkSize = 0x10000;
|
|
5
6
|
/**
|
|
6
7
|
* Provides a consistent and easy to use internal API.
|
|
7
8
|
* Default implementations for `exists` and `existsSync` are included.
|
|
@@ -97,17 +98,22 @@ export class FileSystem {
|
|
|
97
98
|
* @privateRemarks The default implementation of `streamRead` uses "chunked" `read`s
|
|
98
99
|
*/
|
|
99
100
|
streamRead(path, options) {
|
|
101
|
+
const { chunkSize = defaultChunkSize } = options;
|
|
102
|
+
let offset = options.start ?? 0;
|
|
103
|
+
let end = options.end;
|
|
100
104
|
return new ReadableStream({
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
const buffer = new Uint8Array(bytesRead);
|
|
107
|
-
await this.read(path, buffer, offset, offset + bytesRead).catch(controller.error.bind(controller));
|
|
108
|
-
controller.enqueue(buffer);
|
|
105
|
+
pull: async (controller) => {
|
|
106
|
+
end ??= (await this.stat(path)).size;
|
|
107
|
+
if (offset >= end) {
|
|
108
|
+
controller.close();
|
|
109
|
+
return;
|
|
109
110
|
}
|
|
110
|
-
|
|
111
|
+
const buffer = new Uint8Array(Math.min(chunkSize, end - offset));
|
|
112
|
+
await this.read(path, buffer, offset, offset + buffer.byteLength);
|
|
113
|
+
offset += buffer.byteLength;
|
|
114
|
+
controller.enqueue(buffer);
|
|
115
|
+
if (offset >= end)
|
|
116
|
+
controller.close();
|
|
111
117
|
},
|
|
112
118
|
type: 'bytes',
|
|
113
119
|
});
|
package/dist/node/streams.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
/// <reference path="../../types/readable-stream.d.ts" preserve="true" />
|
|
2
1
|
import type { Abortable } from 'node:events';
|
|
3
2
|
import type * as fs from 'node:fs';
|
|
4
3
|
import type { CreateReadStreamOptions, CreateWriteStreamOptions } from 'node:fs/promises';
|
|
5
4
|
import type { Callback } from '../utils.js';
|
|
6
5
|
import type { FileHandle } from './promises.js';
|
|
7
|
-
import { Readable
|
|
6
|
+
import { Readable } from '@zenfs/streams/readable';
|
|
7
|
+
import { Writable } from '@zenfs/streams/writable';
|
|
8
8
|
interface FSImplementation {
|
|
9
9
|
open?: (...args: unknown[]) => unknown;
|
|
10
10
|
close?: (...args: unknown[]) => unknown;
|
package/dist/node/streams.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { eos } from '@zenfs/streams/end-of-stream';
|
|
2
|
+
import { Readable } from '@zenfs/streams/readable';
|
|
3
|
+
import { Writable } from '@zenfs/streams/writable';
|
|
1
4
|
import { Errno, Exception, UV } from 'kerium';
|
|
2
5
|
import { warn } from 'kerium/log';
|
|
3
|
-
import { finished, Readable, Writable } from 'readable-stream';
|
|
4
6
|
/**
|
|
5
7
|
* A ReadStream implementation that wraps an underlying global ReadableStream.
|
|
6
8
|
*/
|
|
@@ -16,7 +18,11 @@ export class ReadStream extends Readable {
|
|
|
16
18
|
.then(handle => {
|
|
17
19
|
this._path = handle['vfs'].path;
|
|
18
20
|
// `end` is inclusive for streams, but the internal range is not
|
|
19
|
-
const internal = handle.readableWebStream({
|
|
21
|
+
const internal = handle.readableWebStream({
|
|
22
|
+
start: opts.start,
|
|
23
|
+
end: opts.end === undefined ? undefined : opts.end + 1,
|
|
24
|
+
chunkSize: opts.highWaterMark,
|
|
25
|
+
});
|
|
20
26
|
this.reader = internal.getReader();
|
|
21
27
|
this.pending = false;
|
|
22
28
|
})
|
|
@@ -46,7 +52,7 @@ export class ReadStream extends Readable {
|
|
|
46
52
|
* Closing before the stream has finished reports `ERR_STREAM_PREMATURE_CLOSE`.
|
|
47
53
|
*/
|
|
48
54
|
close(callback = () => null) {
|
|
49
|
-
|
|
55
|
+
eos(this, error => callback(error ?? null));
|
|
50
56
|
this.destroy();
|
|
51
57
|
}
|
|
52
58
|
get path() {
|
package/dist/vfs/file.js
CHANGED
|
@@ -93,8 +93,8 @@ import { _chown, InodeFlags, isBlockDevice, isCharacterDevice } from '../interna
|
|
|
93
93
|
import '../polyfills.js';
|
|
94
94
|
import { validateFD } from '../utils.js';
|
|
95
95
|
import { cacheOf } from './vcache.js';
|
|
96
|
-
/**
|
|
97
|
-
const streamChunkSize =
|
|
96
|
+
/** Default bytes per chunk for streamed reads, matching Node's default stream `highWaterMark` */
|
|
97
|
+
const streamChunkSize = 0x10000;
|
|
98
98
|
/**
|
|
99
99
|
* @internal
|
|
100
100
|
*/
|
|
@@ -608,19 +608,25 @@ let Handle = (() => {
|
|
|
608
608
|
if (this.closed)
|
|
609
609
|
throw UV('EBADF', 'streamRead', this.path);
|
|
610
610
|
const { vnode } = this;
|
|
611
|
+
const { chunkSize = streamChunkSize } = options;
|
|
612
|
+
let offset = options.start ?? 0;
|
|
613
|
+
let end = options.end;
|
|
611
614
|
return new ReadableStream({
|
|
612
|
-
async
|
|
615
|
+
async pull(controller) {
|
|
613
616
|
const env_15 = { stack: [], error: void 0, hasError: false };
|
|
614
617
|
try {
|
|
615
618
|
const _ = __addDisposableResource(env_15, await vnode.lock('ro'), false);
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
await vnode.read(buffer, offset, offset + bytesRead).catch(controller.error.bind(controller));
|
|
621
|
-
controller.enqueue(buffer);
|
|
619
|
+
end ??= vnode.inode.size;
|
|
620
|
+
if (offset >= end) {
|
|
621
|
+
controller.close();
|
|
622
|
+
return;
|
|
622
623
|
}
|
|
623
|
-
|
|
624
|
+
const buffer = new Uint8Array(Math.min(chunkSize, end - offset));
|
|
625
|
+
await vnode.read(buffer, offset, offset + buffer.byteLength);
|
|
626
|
+
offset += buffer.byteLength;
|
|
627
|
+
controller.enqueue(buffer);
|
|
628
|
+
if (offset >= end)
|
|
629
|
+
controller.close();
|
|
624
630
|
}
|
|
625
631
|
catch (e_15) {
|
|
626
632
|
env_15.error = e_15;
|
package/dist/vfs/vnode.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
2
|
import { RwLockable } from 'kerium/locks';
|
|
3
|
-
import { Resource } from 'utilium/cache';
|
|
3
|
+
import { addRange, Resource } from 'utilium/cache';
|
|
4
4
|
import { InodeFlags, isBlockDevice, isCharacterDevice } from '../internal/inode.js';
|
|
5
5
|
/**
|
|
6
6
|
* A VFS-level node, roughly equivalent to Linux's in-memory `struct inode`.
|
|
@@ -57,17 +57,7 @@ export class VNode extends RwLockable {
|
|
|
57
57
|
}
|
|
58
58
|
/** Add a range to `dirtyRanges`, merging overlapping and adjacent ranges */
|
|
59
59
|
markDirty(start, end) {
|
|
60
|
-
this.dirtyRanges
|
|
61
|
-
this.dirtyRanges.sort((a, b) => a.start - b.start);
|
|
62
|
-
const merged = [];
|
|
63
|
-
for (const range of this.dirtyRanges) {
|
|
64
|
-
const last = merged.at(-1);
|
|
65
|
-
if (last && range.start <= last.end)
|
|
66
|
-
last.end = Math.max(last.end, range.end);
|
|
67
|
-
else
|
|
68
|
-
merged.push(range);
|
|
69
|
-
}
|
|
70
|
-
this.dirtyRanges = merged;
|
|
60
|
+
addRange(this.dirtyRanges, start, end);
|
|
71
61
|
}
|
|
72
62
|
/**
|
|
73
63
|
* Copy `[start, end)` from the cache into `buffer`.
|
|
@@ -75,12 +65,10 @@ export class VNode extends RwLockable {
|
|
|
75
65
|
*/
|
|
76
66
|
copyFromCache(buffer, start, end) {
|
|
77
67
|
let pos = start;
|
|
78
|
-
for (const region of this.data.
|
|
68
|
+
for (const region of this.data.regionsIn(start, end)) {
|
|
79
69
|
const regionEnd = region.offset + region.data.byteLength;
|
|
80
70
|
if (regionEnd <= pos)
|
|
81
71
|
continue;
|
|
82
|
-
if (region.offset >= end)
|
|
83
|
-
break;
|
|
84
72
|
if (region.offset > pos) {
|
|
85
73
|
buffer.fill(0, pos - start, region.offset - start);
|
|
86
74
|
pos = region.offset;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zenfs/core",
|
|
3
|
-
"version": "2.7.
|
|
3
|
+
"version": "2.7.4",
|
|
4
4
|
"description": "A filesystem, anywhere",
|
|
5
5
|
"funding": {
|
|
6
6
|
"type": "individual",
|
|
@@ -69,18 +69,19 @@
|
|
|
69
69
|
"lint": "eslint src tests",
|
|
70
70
|
"test": "npx -s zenfs-test --clean; npx -s zenfs-test -abcpn; tests/fetch/run.sh; npx -s zenfs-test --report-only",
|
|
71
71
|
"test:native": "npx -s zenfs-test tests/native-setup.ts --verbose",
|
|
72
|
+
"test:perf": "zbench",
|
|
72
73
|
"build": "tsc --build",
|
|
73
74
|
"dev": "tsc --build --watch",
|
|
74
75
|
"prepublishOnly": "npm run build"
|
|
75
76
|
},
|
|
76
77
|
"dependencies": {
|
|
77
78
|
"@types/node": "^26.0.0",
|
|
79
|
+
"@zenfs/streams": "^1.0.0",
|
|
78
80
|
"buffer": "^6.0.3",
|
|
79
81
|
"eventemitter3": "^5.0.1",
|
|
80
82
|
"kerium": "^1.5.0",
|
|
81
83
|
"memium": "^1.0.0",
|
|
82
|
-
"
|
|
83
|
-
"utilium": "^3.0.0"
|
|
84
|
+
"utilium": "^3.7.0"
|
|
84
85
|
},
|
|
85
86
|
"devDependencies": {
|
|
86
87
|
"@eslint/js": "^10.0.1",
|
|
@@ -91,7 +92,8 @@
|
|
|
91
92
|
"prettier": "^3.2.5",
|
|
92
93
|
"typedoc": "^0.28.18",
|
|
93
94
|
"typescript": "^6.0.0",
|
|
94
|
-
"typescript-eslint": "^8.58.0"
|
|
95
|
+
"typescript-eslint": "^8.58.0",
|
|
96
|
+
"zbench-js": "^0.1.0"
|
|
95
97
|
},
|
|
96
98
|
"prettier": {
|
|
97
99
|
"singleQuote": true,
|
package/tests/fs/streams.test.ts
CHANGED
|
@@ -141,4 +141,64 @@ suite('Streams', config('streams'), () => {
|
|
|
141
141
|
// Closing invalidates the descriptor, so it fails range validation
|
|
142
142
|
assert.throws(() => fileHandle.createWriteStream(), { code: 'ERR_OUT_OF_RANGE' });
|
|
143
143
|
});
|
|
144
|
+
|
|
145
|
+
// A file large enough to span several chunks, so the read path has to stitch
|
|
146
|
+
// them back together and the vnode cache has to grow to hold them.
|
|
147
|
+
const large = Buffer.alloc(200 * 1024);
|
|
148
|
+
for (let i = 0; i < large.length; i++) large[i] = (i * 31 + 7) & 0xff;
|
|
149
|
+
|
|
150
|
+
test('createReadStream reads a multi-chunk file byte for byte', async () => {
|
|
151
|
+
await fs.promises.writeFile('large.bin', large);
|
|
152
|
+
|
|
153
|
+
const chunks: Uint8Array[] = [];
|
|
154
|
+
for await (const chunk of fs.createReadStream('large.bin')) chunks.push(chunk as Uint8Array);
|
|
155
|
+
|
|
156
|
+
assert.deepEqual(Buffer.concat(chunks), large);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test('createReadStream honors highWaterMark as the chunk size', async () => {
|
|
160
|
+
await fs.promises.writeFile('large.bin', large);
|
|
161
|
+
|
|
162
|
+
const highWaterMark = 64 * 1024;
|
|
163
|
+
const sizes: number[] = [];
|
|
164
|
+
for await (const chunk of fs.createReadStream('large.bin', { highWaterMark })) sizes.push((chunk as Uint8Array).byteLength);
|
|
165
|
+
|
|
166
|
+
// Every chunk but the last is a full highWaterMark, and they sum to the file
|
|
167
|
+
assert.deepEqual(sizes.slice(0, -1), Array(sizes.length - 1).fill(highWaterMark));
|
|
168
|
+
assert.equal(
|
|
169
|
+
sizes.reduce((a, b) => a + b, 0),
|
|
170
|
+
large.length
|
|
171
|
+
);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test('createReadStream with start and end spanning chunks', async () => {
|
|
175
|
+
await fs.promises.writeFile('large.bin', large);
|
|
176
|
+
|
|
177
|
+
// `end` is inclusive, and this range deliberately starts and stops mid-chunk
|
|
178
|
+
const start = 1000;
|
|
179
|
+
const end = 150_000;
|
|
180
|
+
const chunks: Uint8Array[] = [];
|
|
181
|
+
for await (const chunk of fs.createReadStream('large.bin', { start, end, highWaterMark: 8192 })) chunks.push(chunk as Uint8Array);
|
|
182
|
+
|
|
183
|
+
assert.deepEqual(Buffer.concat(chunks), large.subarray(start, end + 1));
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test('a read stream does not read past the end of the file', async () => {
|
|
187
|
+
await fs.promises.writeFile('small.bin', large.subarray(0, 10));
|
|
188
|
+
|
|
189
|
+
const chunks: Uint8Array[] = [];
|
|
190
|
+
for await (const chunk of fs.createReadStream('small.bin', { highWaterMark: 64 * 1024 })) chunks.push(chunk as Uint8Array);
|
|
191
|
+
|
|
192
|
+
assert.equal(chunks.length, 1);
|
|
193
|
+
assert.deepEqual(chunks[0], large.subarray(0, 10));
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test('a read stream over an empty file ends without a chunk', async () => {
|
|
197
|
+
await fs.promises.writeFile('empty.bin', '');
|
|
198
|
+
|
|
199
|
+
const chunks: unknown[] = [];
|
|
200
|
+
for await (const chunk of fs.createReadStream('empty.bin')) chunks.push(chunk);
|
|
201
|
+
|
|
202
|
+
assert.deepEqual(chunks, []);
|
|
203
|
+
});
|
|
144
204
|
});
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
// Sequential appends into one region. Every write extends the same region, so a region that
|
|
3
|
+
// allocates exactly what it needs copies the whole file on each append.
|
|
4
|
+
import type { FileHandle } from '@zenfs/core/promises';
|
|
5
|
+
import { fs, mount, piece } from './common.ts';
|
|
6
|
+
|
|
7
|
+
interface Config {
|
|
8
|
+
writes: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
interface State {
|
|
12
|
+
data: Buffer;
|
|
13
|
+
handle?: FileHandle;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export async function setup(): Promise<State> {
|
|
17
|
+
await mount('perf-append');
|
|
18
|
+
return { data: Buffer.alloc(piece, 0x11) };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export async function before(config: Config, state: State): Promise<void> {
|
|
22
|
+
state.handle = await fs.promises.open('/a', 'w');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function test(config: Config, state: State) {
|
|
26
|
+
for (let i = 0; i < config.writes; i++) await state.handle!.write(state.data, 0, state.data.length, i * piece);
|
|
27
|
+
|
|
28
|
+
return { writes: config.writes, bytes: config.writes * piece };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function after(config: Config, state: State): Promise<void> {
|
|
32
|
+
await state.handle!.close();
|
|
33
|
+
await fs.promises.rm('/a');
|
|
34
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
import { configureSingle, fs, InMemory } from '@zenfs/core';
|
|
3
|
+
|
|
4
|
+
export { fs };
|
|
5
|
+
|
|
6
|
+
export const KB = 1024;
|
|
7
|
+
export const MB = 1024 * 1024;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A piece and the stride between pieces, chosen so consecutive writes never merge into one region:
|
|
11
|
+
* the 12 KB hole between them is wider than `regionGapThreshold`.
|
|
12
|
+
*/
|
|
13
|
+
export const piece = 4 * KB;
|
|
14
|
+
export const stride = 16 * KB;
|
|
15
|
+
|
|
16
|
+
/** A fresh in-memory filesystem, so no configuration inherits another's cached vnodes. */
|
|
17
|
+
export async function mount(name: string): Promise<void> {
|
|
18
|
+
await configureSingle({ backend: InMemory, name });
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Split a buffer the way a stream writer feeds it. */
|
|
22
|
+
export function chunks(buffer: Buffer, size: number): Buffer[] {
|
|
23
|
+
const out: Buffer[] = [];
|
|
24
|
+
for (let i = 0; i < buffer.length; i += size) out.push(buffer.subarray(i, Math.min(i + size, buffer.length)));
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Give the event loop a moment, so work queued by the previous phase never lands in the timed one. */
|
|
29
|
+
export function settle(): Promise<void> {
|
|
30
|
+
return new Promise(resolve => setTimeout(resolve, 50));
|
|
31
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
{
|
|
2
|
+
"flags": {
|
|
3
|
+
"prep": ["writeFile", "stream"]
|
|
4
|
+
},
|
|
5
|
+
"tests": [
|
|
6
|
+
{
|
|
7
|
+
"path": "stream-read.ts",
|
|
8
|
+
"name": "Read with createReadStream",
|
|
9
|
+
"flags": ["prep"],
|
|
10
|
+
"measure": {
|
|
11
|
+
"bytes": { "throughput": "s", "aggregate_throughput": "s", "unit": "MB" },
|
|
12
|
+
"files": { "cost": true, "unit": "file" }
|
|
13
|
+
},
|
|
14
|
+
"configurations": [
|
|
15
|
+
{ "name": "4K x2000", "value": { "size": 4096, "count": 2000, "hwm": 65536 } },
|
|
16
|
+
{ "name": "256K x200", "value": { "size": 262144, "count": 200, "hwm": 65536 } },
|
|
17
|
+
{ "name": "8M x8", "cpu": 1, "mem": 1, "value": { "size": 8388608, "count": 8, "hwm": 65536 } },
|
|
18
|
+
{
|
|
19
|
+
"name": "8M x8, hwm 1M",
|
|
20
|
+
"cpu": 1,
|
|
21
|
+
"mem": 1,
|
|
22
|
+
"value": { "size": 8388608, "count": 8, "hwm": 1048576 }
|
|
23
|
+
}
|
|
24
|
+
]
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"path": "stream-write.ts",
|
|
28
|
+
"name": "Write with createWriteStream",
|
|
29
|
+
"measure": {
|
|
30
|
+
"bytes": { "throughput": "s", "aggregate_throughput": "s", "unit": "MB" },
|
|
31
|
+
"files": { "cost": true, "unit": "file" }
|
|
32
|
+
},
|
|
33
|
+
"configurations": [
|
|
34
|
+
{ "name": "4K x2000", "value": { "size": 4096, "count": 2000, "hwm": 65536 } },
|
|
35
|
+
{ "name": "256K x200", "value": { "size": 262144, "count": 200, "hwm": 65536 } },
|
|
36
|
+
{ "name": "8M x8", "cpu": 1, "mem": 1, "value": { "size": 8388608, "count": 8, "hwm": 65536 } },
|
|
37
|
+
{
|
|
38
|
+
"name": "8M x8, hwm 1M",
|
|
39
|
+
"cpu": 1,
|
|
40
|
+
"mem": 1,
|
|
41
|
+
"value": { "size": 8388608, "count": 8, "hwm": 1048576 }
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"path": "fragmented-read.ts",
|
|
47
|
+
"name": "Read across many cached regions",
|
|
48
|
+
"measure": {
|
|
49
|
+
"reads": { "cost": true, "aggregate_cost": true, "unit": "read" }
|
|
50
|
+
},
|
|
51
|
+
"configurations": [
|
|
52
|
+
{ "name": "128 regions", "value": { "regions": 128, "reads": 4000 } },
|
|
53
|
+
{ "name": "512 regions", "value": { "regions": 512, "reads": 4000 } },
|
|
54
|
+
{ "name": "2048 regions", "cpu": 1, "value": { "regions": 2048, "reads": 4000 } },
|
|
55
|
+
{ "name": "8192 regions", "cpu": 2, "mem": 1, "value": { "regions": 8192, "reads": 4000 } }
|
|
56
|
+
]
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"path": "fragmented-write.ts",
|
|
60
|
+
"name": "Scattered writes into one file",
|
|
61
|
+
"measure": {
|
|
62
|
+
"writes": { "cost": true, "aggregate_cost": true, "unit": "write" }
|
|
63
|
+
},
|
|
64
|
+
"configurations": [
|
|
65
|
+
{ "name": "512 writes", "value": { "writes": 512 } },
|
|
66
|
+
{ "name": "2048 writes", "cpu": 1, "value": { "writes": 2048 } },
|
|
67
|
+
{ "name": "8192 writes", "cpu": 2, "mem": 1, "value": { "writes": 8192 } }
|
|
68
|
+
]
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"path": "append.ts",
|
|
72
|
+
"name": "Append 4 KB at a time",
|
|
73
|
+
"measure": {
|
|
74
|
+
"bytes": { "throughput": "s", "unit": "MB" },
|
|
75
|
+
"writes": { "cost": true, "aggregate_cost": true, "unit": "write" }
|
|
76
|
+
},
|
|
77
|
+
"configurations": [
|
|
78
|
+
{ "name": "500 appends", "value": { "writes": 500 } },
|
|
79
|
+
{ "name": "2000 appends", "value": { "writes": 2000 } },
|
|
80
|
+
{ "name": "8000 appends", "cpu": 1, "value": { "writes": 8000 } }
|
|
81
|
+
]
|
|
82
|
+
}
|
|
83
|
+
]
|
|
84
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
// Reads spread across a file with many disjoint cached regions.
|
|
3
|
+
|
|
4
|
+
import type { FileHandle } from '@zenfs/core/promises';
|
|
5
|
+
import { fs, mount, piece, stride } from './common.ts';
|
|
6
|
+
|
|
7
|
+
interface Config {
|
|
8
|
+
/** Cached regions to build, one per scattered write */
|
|
9
|
+
regions: number;
|
|
10
|
+
reads: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
interface State {
|
|
14
|
+
handle: FileHandle;
|
|
15
|
+
target: Buffer;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export async function setup(config: Config): Promise<State> {
|
|
19
|
+
await mount('perf-fragmented-read');
|
|
20
|
+
|
|
21
|
+
await fs.promises.writeFile('/f', Buffer.alloc(config.regions * stride));
|
|
22
|
+
const handle = await fs.promises.open('/f', 'r+');
|
|
23
|
+
|
|
24
|
+
const data = Buffer.alloc(piece, 0xcd);
|
|
25
|
+
for (let i = 0; i < config.regions; i++) await handle.write(data, 0, data.length, i * stride);
|
|
26
|
+
|
|
27
|
+
return { handle, target: Buffer.allocUnsafe(piece) };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function test(config: Config, state: State) {
|
|
31
|
+
// Backwards, so a scan that starts at region zero pays its worst case on every read
|
|
32
|
+
for (let i = 0; i < config.reads; i++)
|
|
33
|
+
await state.handle.read(state.target, 0, state.target.length, (config.regions - 1 - (i % config.regions)) * stride);
|
|
34
|
+
|
|
35
|
+
return { reads: config.reads, bytes: config.reads * piece };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function teardown(config: Config, state: State): Promise<void> {
|
|
39
|
+
await state.handle.close();
|
|
40
|
+
await fs.promises.rm('/f');
|
|
41
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
// Writes that each land in their own region, so the region list grows by one per write.
|
|
3
|
+
// Anything that walks the whole list — or re-sorts the dirty ranges — per write shows up as
|
|
4
|
+
// quadratic here.
|
|
5
|
+
import type { FileHandle } from '@zenfs/core/promises';
|
|
6
|
+
import { fs, mount, piece, stride } from './common.ts';
|
|
7
|
+
|
|
8
|
+
interface Config {
|
|
9
|
+
writes: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
interface State {
|
|
13
|
+
data: Buffer;
|
|
14
|
+
handle?: FileHandle;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function setup(): Promise<State> {
|
|
18
|
+
await mount('perf-fragmented-write');
|
|
19
|
+
return { data: Buffer.alloc(piece, 0xcd) };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** A fresh file each iteration: the point is the cost of building the region list, not of reusing it. */
|
|
23
|
+
export async function before(config: Config, state: State): Promise<void> {
|
|
24
|
+
await fs.promises.writeFile('/f', Buffer.alloc(config.writes * stride));
|
|
25
|
+
state.handle = await fs.promises.open('/f', 'r+');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export async function test(config: Config, state: State) {
|
|
29
|
+
for (let i = 0; i < config.writes; i++) await state.handle!.write(state.data, 0, state.data.length, i * stride);
|
|
30
|
+
|
|
31
|
+
return { writes: config.writes, bytes: config.writes * piece };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export async function after(config: Config, state: State): Promise<void> {
|
|
35
|
+
await state.handle!.close();
|
|
36
|
+
await fs.promises.rm('/f');
|
|
37
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
// Reading through `createReadStream`: the VFS cache lookup, `copyFromCache`, and the stream's own
|
|
3
|
+
// chunking, with the files written outside the timed window.
|
|
4
|
+
import { pipeline } from 'node:stream/promises';
|
|
5
|
+
import { chunks, fs, mount, settle } from './common.ts';
|
|
6
|
+
|
|
7
|
+
interface Config {
|
|
8
|
+
size: number;
|
|
9
|
+
count: number;
|
|
10
|
+
/** Stream highWaterMark, which is also the read chunk size */
|
|
11
|
+
hwm: number;
|
|
12
|
+
prep: 'writeFile' | 'stream';
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
interface State {
|
|
16
|
+
buffer: Buffer;
|
|
17
|
+
parts: Buffer[];
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export async function setup(config: Config): Promise<State> {
|
|
21
|
+
await mount('perf-stream-read');
|
|
22
|
+
const buffer = Buffer.alloc(config.size, 0xab);
|
|
23
|
+
return { buffer, parts: chunks(buffer, config.hwm) };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function before(config: Config, state: State): Promise<void> {
|
|
27
|
+
for (let i = 0; i < config.count; i++) {
|
|
28
|
+
if (config.prep == 'stream') await pipeline(state.parts, fs.createWriteStream(`/w${i}`, { highWaterMark: config.hwm }));
|
|
29
|
+
else await fs.promises.writeFile(`/w${i}`, state.buffer);
|
|
30
|
+
}
|
|
31
|
+
await settle();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export async function test(config: Config) {
|
|
35
|
+
let bytes = 0;
|
|
36
|
+
|
|
37
|
+
for (let i = 0; i < config.count; i++) {
|
|
38
|
+
for await (const chunk of fs.createReadStream(`/w${i}`, { highWaterMark: config.hwm })) bytes += (chunk as Buffer).length;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (bytes != config.size * config.count) throw new Error(`short read: ${bytes} of ${config.size * config.count}`);
|
|
42
|
+
|
|
43
|
+
return { bytes, files: config.count };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export async function after(config: Config): Promise<void> {
|
|
47
|
+
for (let i = 0; i < config.count; i++) await fs.promises.rm(`/w${i}`);
|
|
48
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
// Writing through `createWriteStream`: how the cache grows as a file is appended to one chunk at a
|
|
3
|
+
// time. A region that reallocates exactly on every append makes this quadratic in the file's size.
|
|
4
|
+
import { pipeline } from 'node:stream/promises';
|
|
5
|
+
import { chunks, fs, mount, settle } from './common.ts';
|
|
6
|
+
|
|
7
|
+
interface Config {
|
|
8
|
+
size: number;
|
|
9
|
+
count: number;
|
|
10
|
+
hwm: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
interface State {
|
|
14
|
+
parts: Buffer[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function setup(config: Config): Promise<State> {
|
|
18
|
+
await mount('perf-stream-write');
|
|
19
|
+
return { parts: chunks(Buffer.alloc(config.size, 0xab), config.hwm) };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export async function test(config: Config, state: State) {
|
|
23
|
+
for (let i = 0; i < config.count; i++) await pipeline(state.parts, fs.createWriteStream(`/w${i}`, { highWaterMark: config.hwm }));
|
|
24
|
+
|
|
25
|
+
return { bytes: config.size * config.count, files: config.count };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export async function after(config: Config): Promise<void> {
|
|
29
|
+
for (let i = 0; i < config.count; i++) await fs.promises.rm(`/w${i}`);
|
|
30
|
+
await settle();
|
|
31
|
+
}
|
package/tests/setup.ts
CHANGED
|
@@ -8,7 +8,7 @@ export const data = join(import.meta.dirname, 'data');
|
|
|
8
8
|
|
|
9
9
|
export const tmp = join(import.meta.dirname, 'tmp');
|
|
10
10
|
|
|
11
|
-
if (!existsSync(tmp)) mkdirSync(tmp);
|
|
11
|
+
if (!existsSync(tmp)) mkdirSync(tmp, { recursive: true });
|
|
12
12
|
|
|
13
13
|
export async function copyAsync(_path: string, fs: NodeFS = _fs): Promise<void> {
|
|
14
14
|
const path = relative(data, _path) || '/';
|
package/types/README.md
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
This directory contains declaration files. Due to numerous TypeScript shenanigans, placing them into `src` is a really bad idea. Instead, we put them here.
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
@types/readable-stream has *many* problems:
|
|
3
|
-
https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71923
|
|
4
|
-
https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71307
|
|
5
|
-
https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71083
|
|
6
|
-
https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/66049
|
|
7
|
-
... and many more
|
|
8
|
-
|
|
9
|
-
This allows us to bypass those problems.
|
|
10
|
-
|
|
11
|
-
Warning: Do not install @types/readable-stream alongside this package!
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
/// <reference types="node" preserve="true" />
|
|
15
|
-
declare module 'readable-stream' {
|
|
16
|
-
import { finished, Readable, Writable } from 'node:stream';
|
|
17
|
-
export { finished, Readable, Writable };
|
|
18
|
-
}
|