@zenfs/core 2.7.6 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/backends/index.d.ts +3 -1
- package/dist/backends/index.js +3 -1
- package/dist/backends/single_buffer/backend.d.ts +42 -0
- package/dist/backends/single_buffer/backend.js +204 -0
- package/dist/backends/{single_buffer.d.ts → single_buffer/v1.d.ts} +24 -34
- package/dist/backends/{single_buffer.js → single_buffer/v1.js} +10 -26
- package/dist/backends/single_buffer/v2.d.ts +257 -0
- package/dist/backends/single_buffer/v2.js +1283 -0
- package/dist/backends/store/fs.d.ts +1 -1
- package/dist/backends/store/fs.js +35 -18
- package/dist/backends/store/map.d.ts +6 -6
- package/dist/backends/store/map.js +6 -6
- package/dist/backends/store/store.d.ts +18 -14
- package/dist/backends/store/store.js +12 -14
- package/dist/config.d.ts +7 -0
- package/dist/config.js +9 -3
- package/dist/internal/contexts.d.ts +5 -0
- package/dist/internal/contexts.js +7 -1
- package/dist/internal/error.d.ts +5 -0
- package/dist/internal/error.js +10 -0
- package/dist/internal/inode.d.ts +25 -0
- package/dist/internal/inode.js +75 -10
- package/dist/mixins/async.js +47 -30
- package/dist/node/promises.js +23 -124
- package/dist/node/stats.d.ts +1 -1
- package/dist/node/stats.js +4 -1
- package/dist/node/sync.js +20 -51
- package/dist/polyfills.js +16 -0
- package/dist/vfs/async.d.ts +4 -1
- package/dist/vfs/async.js +149 -69
- package/dist/vfs/config.d.ts +10 -1
- package/dist/vfs/config.js +5 -2
- package/dist/vfs/file.js +10 -4
- package/dist/vfs/ioctl.js +2 -2
- package/dist/vfs/shared.d.ts +2 -0
- package/dist/vfs/shared.js +3 -4
- package/dist/vfs/sync.d.ts +4 -1
- package/dist/vfs/sync.js +142 -64
- package/dist/vfs/vcache.d.ts +14 -14
- package/dist/vfs/vcache.js +65 -57
- package/dist/vfs/watchers.d.ts +1 -0
- package/dist/vfs/watchers.js +2 -4
- package/dist/vfs/xattr.js +18 -17
- package/package.json +3 -4
- package/scripts/make-index.js +1 -1
- package/tests/backend/single-buffer-rotate.worker.js +1 -1
- package/tests/backend/single-buffer-v2.test.ts +342 -0
- package/tests/backend/single-buffer-v2.worker.js +51 -0
- package/tests/backend/single-buffer.test.ts +6 -6
- package/tests/backend/single-buffer.worker.js +1 -0
- package/tests/common/async.test.ts +46 -0
- package/tests/common/config.test.ts +1 -1
- package/tests/common/context.test.ts +30 -0
- package/tests/common/file_index.test.ts +1 -1
- package/tests/common/mounts.test.ts +12 -0
- package/tests/common/vcache.test.ts +13 -0
- package/tests/fs/concurrency.test.ts +3 -3
- package/tests/fs/links.test.ts +76 -0
- package/tests/fs/permissions.test.ts +253 -0
- package/tests/fs/stat.test.ts +1 -1
- package/tests/fs/streams.test.ts +1 -1
- package/tests/fs/watch.test.ts +14 -0
- package/tests/fs/xattr.test.ts +10 -0
- package/tests/perf/config.json +31 -1
- package/tests/perf/single-buffer-random.ts +38 -0
- package/tests/perf/single-buffer-read.ts +28 -0
- package/tests/perf/single-buffer-write.ts +41 -0
- package/tests/setup/single-buffer-v2.ts +10 -0
- package/tests/setup/single-buffer.ts +1 -0
package/dist/vfs/xattr.js
CHANGED
|
@@ -5,7 +5,8 @@ import { Attributes, hasAccess } from '../internal/inode.js';
|
|
|
5
5
|
import { normalizePath } from '../utils.js';
|
|
6
6
|
import { checkAccess } from './config.js';
|
|
7
7
|
import { R_OK, W_OK } from '../constants.js';
|
|
8
|
-
import {
|
|
8
|
+
import { resolve as resolveAsync } from './async.js';
|
|
9
|
+
import { resolve as resolveSync } from './sync.js';
|
|
9
10
|
/** Attributes outside the `user` namespace that are supported. */
|
|
10
11
|
export const allowedRestrictedNames = new Set();
|
|
11
12
|
/**
|
|
@@ -19,9 +20,9 @@ function checkName($, name, path, syscall) {
|
|
|
19
20
|
}
|
|
20
21
|
export async function get(path, name, opt = {}) {
|
|
21
22
|
path = normalizePath.call(this, path);
|
|
22
|
-
const {
|
|
23
|
+
const { cache, path: resolved } = await resolveAsync(this, path, opt.noFollow);
|
|
23
24
|
checkName(this, name, path, 'xattr.get');
|
|
24
|
-
const inode = await
|
|
25
|
+
const inode = await cache.stat(resolved).catch(rethrow('xattr.get', path));
|
|
25
26
|
if (checkAccess && !hasAccess(this, inode, R_OK))
|
|
26
27
|
throw UV('EACCES', 'xattr.get', path);
|
|
27
28
|
inode.attributes ??= new Attributes();
|
|
@@ -34,10 +35,10 @@ export async function get(path, name, opt = {}) {
|
|
|
34
35
|
export function getSync(path, name, opt = {}) {
|
|
35
36
|
path = normalizePath.call(this, path);
|
|
36
37
|
checkName(this, name, path, 'xattr.get');
|
|
37
|
-
const {
|
|
38
|
+
const { cache, path: resolved } = resolveSync(this, path, opt.noFollow);
|
|
38
39
|
let inode;
|
|
39
40
|
try {
|
|
40
|
-
inode =
|
|
41
|
+
inode = cache.statSync(resolved);
|
|
41
42
|
}
|
|
42
43
|
catch (e) {
|
|
43
44
|
throw setUVMessage(Object.assign(e, { path }));
|
|
@@ -61,9 +62,9 @@ export function getSync(path, name, opt = {}) {
|
|
|
61
62
|
*/
|
|
62
63
|
export async function set(path, name, value, opt = {}) {
|
|
63
64
|
path = normalizePath.call(this, path);
|
|
64
|
-
const { fs, path: resolved } =
|
|
65
|
+
const { fs, cache, path: resolved } = await resolveAsync(this, path, opt.noFollow);
|
|
65
66
|
checkName(this, name, path, 'xattr.set');
|
|
66
|
-
const inode = await
|
|
67
|
+
const inode = await cache.stat(resolved).catch(rethrow('xattr.set', path));
|
|
67
68
|
if (checkAccess && !hasAccess(this, inode, W_OK))
|
|
68
69
|
throw UV('EACCES', 'xattr.set', path);
|
|
69
70
|
inode.attributes ??= new Attributes();
|
|
@@ -85,11 +86,11 @@ export async function set(path, name, value, opt = {}) {
|
|
|
85
86
|
*/
|
|
86
87
|
export function setSync(path, name, value, opt = {}) {
|
|
87
88
|
path = normalizePath.call(this, path);
|
|
88
|
-
const { fs, path: resolved } =
|
|
89
|
+
const { fs, cache, path: resolved } = resolveSync(this, path, opt.noFollow);
|
|
89
90
|
checkName(this, name, path, 'xattr.set');
|
|
90
91
|
let inode;
|
|
91
92
|
try {
|
|
92
|
-
inode =
|
|
93
|
+
inode = cache.statSync(resolved);
|
|
93
94
|
}
|
|
94
95
|
catch (e) {
|
|
95
96
|
throw setUVMessage(Object.assign(e, { path }));
|
|
@@ -118,9 +119,9 @@ export function setSync(path, name, value, opt = {}) {
|
|
|
118
119
|
*/
|
|
119
120
|
export async function remove(path, name) {
|
|
120
121
|
path = normalizePath.call(this, path);
|
|
121
|
-
const { fs, path: resolved } =
|
|
122
|
+
const { fs, cache, path: resolved } = await resolveAsync(this, path);
|
|
122
123
|
checkName(this, name, path, 'xattr.remove');
|
|
123
|
-
const inode = await
|
|
124
|
+
const inode = await cache.stat(resolved).catch(rethrow('xattr.remove', path));
|
|
124
125
|
if (checkAccess && !hasAccess(this, inode, W_OK))
|
|
125
126
|
throw UV('EACCES', 'xattr.remove', path);
|
|
126
127
|
inode.attributes ??= new Attributes();
|
|
@@ -138,11 +139,11 @@ export async function remove(path, name) {
|
|
|
138
139
|
*/
|
|
139
140
|
export function removeSync(path, name) {
|
|
140
141
|
path = normalizePath.call(this, path);
|
|
141
|
-
const { fs, path: resolved } =
|
|
142
|
+
const { fs, cache, path: resolved } = resolveSync(this, path);
|
|
142
143
|
checkName(this, name, path, 'xattr.remove');
|
|
143
144
|
let inode;
|
|
144
145
|
try {
|
|
145
|
-
inode =
|
|
146
|
+
inode = cache.statSync(resolved);
|
|
146
147
|
}
|
|
147
148
|
catch (e) {
|
|
148
149
|
throw setUVMessage(Object.assign(e, { path }));
|
|
@@ -169,8 +170,8 @@ export function removeSync(path, name) {
|
|
|
169
170
|
*/
|
|
170
171
|
export async function list(path) {
|
|
171
172
|
path = normalizePath.call(this, path);
|
|
172
|
-
const {
|
|
173
|
-
const inode = await
|
|
173
|
+
const { cache, path: resolved } = await resolveAsync(this, path);
|
|
174
|
+
const inode = await cache.stat(resolved).catch(rethrow('xattr.list', path));
|
|
174
175
|
if (!inode.attributes)
|
|
175
176
|
return [];
|
|
176
177
|
return [...inode.attributes.keys()];
|
|
@@ -183,10 +184,10 @@ export async function list(path) {
|
|
|
183
184
|
*/
|
|
184
185
|
export function listSync(path) {
|
|
185
186
|
path = normalizePath.call(this, path);
|
|
186
|
-
const {
|
|
187
|
+
const { cache, path: resolved } = resolveSync(this, path);
|
|
187
188
|
let inode;
|
|
188
189
|
try {
|
|
189
|
-
inode =
|
|
190
|
+
inode = cache.statSync(resolved);
|
|
190
191
|
}
|
|
191
192
|
catch (e) {
|
|
192
193
|
throw setUVMessage(Object.assign(e, { path }));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zenfs/core",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.8.0",
|
|
4
4
|
"description": "A filesystem, anywhere",
|
|
5
5
|
"funding": {
|
|
6
6
|
"type": "individual",
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"spdx:fix": "npx lice -aw src tests/*.ts tests/**/*.test.ts",
|
|
68
68
|
"lint": "eslint src tests",
|
|
69
69
|
"test": "npx -s zenfs-test --clean; npx -s zenfs-test -abcpn; tests/fetch/run.sh; npx -s zenfs-test --report-only",
|
|
70
|
-
"test:native": "npx -s zenfs-test tests/native-setup.ts --verbose",
|
|
70
|
+
"test:native": "npx -s zenfs-test -n tests/native-setup.ts --verbose",
|
|
71
71
|
"test:perf": "zbench",
|
|
72
72
|
"build": "tsc --build",
|
|
73
73
|
"dev": "tsc --build --watch",
|
|
@@ -79,7 +79,7 @@
|
|
|
79
79
|
"buffer": "^6.0.3",
|
|
80
80
|
"eventemitter3": "^5.0.1",
|
|
81
81
|
"kerium": "^1.5.0",
|
|
82
|
-
"memium": "^1.
|
|
82
|
+
"memium": "^1.1.0",
|
|
83
83
|
"utilium": "^3.7.0"
|
|
84
84
|
},
|
|
85
85
|
"devDependencies": {
|
|
@@ -97,7 +97,6 @@
|
|
|
97
97
|
"prettier": {
|
|
98
98
|
"singleQuote": true,
|
|
99
99
|
"useTabs": true,
|
|
100
|
-
"trailingComma": "es5",
|
|
101
100
|
"tabWidth": 4,
|
|
102
101
|
"printWidth": 150,
|
|
103
102
|
"arrowParens": "avoid",
|
package/scripts/make-index.js
CHANGED
|
@@ -94,7 +94,7 @@ function computeEntries(path) {
|
|
|
94
94
|
else if (!seenDevs.has(stats.dev))
|
|
95
95
|
if (options.xdev) {
|
|
96
96
|
console.warn(
|
|
97
|
-
`${styleText('yellowBright', `--xdev: Adding entries from device ${stats.dev} (${path}). You may get duplicate inos which MUST be de-duplicated manually.`)}
|
|
97
|
+
`${styleText('yellowBright', `--xdev: Adding entries from device ${stats.dev} (${path}). You may get duplicate inos which MUST be de-duplicated manually.`)}`,
|
|
98
98
|
);
|
|
99
99
|
seenDevs.add(stats.dev);
|
|
100
100
|
} else {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
2
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
3
|
-
import { SuperBlock } from '../../dist/backends/single_buffer.js';
|
|
3
|
+
import { SuperBlock } from '../../dist/backends/single_buffer/v1.js';
|
|
4
4
|
import { setupLogs } from '../logs.js';
|
|
5
5
|
|
|
6
6
|
setupLogs('<worker>');
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
import { fs, mount, resolveMountConfig, SingleBuffer, vfs } from '@zenfs/core';
|
|
3
|
+
import { SingleBufferStore } from '@zenfs/core/backends/single_buffer/v1';
|
|
4
|
+
import { SingleBufferStore as SingleBufferStoreV2, SuperBlock } from '@zenfs/core/backends/single_buffer/v2';
|
|
5
|
+
import { Mutex } from 'memium/locks';
|
|
6
|
+
import assert from 'node:assert/strict';
|
|
7
|
+
import { randomBytes } from 'node:crypto';
|
|
8
|
+
import { suite, test } from 'node:test';
|
|
9
|
+
import { Worker } from 'worker_threads';
|
|
10
|
+
import { setupLogs } from '../logs.js';
|
|
11
|
+
|
|
12
|
+
setupLogs();
|
|
13
|
+
|
|
14
|
+
function runWorker(workerData: object): Promise<unknown> {
|
|
15
|
+
const worker = new Worker(import.meta.dirname + '/single-buffer-v2.worker.js', { workerData });
|
|
16
|
+
const { promise, resolve, reject } = Promise.withResolvers<unknown>();
|
|
17
|
+
const timeout = setTimeout(() => reject(new Error('worker timed out')), 5000);
|
|
18
|
+
worker.on('error', reject);
|
|
19
|
+
worker.on('message', message => (message instanceof Error ? reject(message) : resolve(message)));
|
|
20
|
+
return promise.finally(() => {
|
|
21
|
+
clearTimeout(timeout);
|
|
22
|
+
void worker.terminate();
|
|
23
|
+
worker.unref();
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
let mounts = 0;
|
|
28
|
+
|
|
29
|
+
/** Mounts a SingleBuffer on a unique mount point */
|
|
30
|
+
async function mountBuffer(buffer: ArrayBufferLike): Promise<[path: string, umount: () => void]> {
|
|
31
|
+
const path = `/sbfs-v2-${mounts++}`;
|
|
32
|
+
mount(path, await resolveMountConfig({ backend: SingleBuffer, buffer, v2: true }));
|
|
33
|
+
return [path, () => vfs.umount(path)];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Bytes not available for new data */
|
|
37
|
+
function used(path: string): number {
|
|
38
|
+
const { blocks, bfree, bsize } = fs.statfsSync(path);
|
|
39
|
+
return (blocks - bfree) * bsize;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
await suite('SingleBuffer v2', () => {
|
|
43
|
+
test('filesystem restoration from original buffer (with same metadata)', async () => {
|
|
44
|
+
const buffer = new ArrayBuffer(0x100000);
|
|
45
|
+
|
|
46
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
47
|
+
fs.writeFileSync(path + '/example.ts', 'console.log("hello world")', 'utf-8');
|
|
48
|
+
const stats = fs.statSync(path + '/example.ts');
|
|
49
|
+
|
|
50
|
+
const [snapshot, umountSnapshot] = await mountBuffer(buffer);
|
|
51
|
+
assert.deepEqual(fs.statSync(snapshot + '/example.ts'), stats);
|
|
52
|
+
|
|
53
|
+
umount();
|
|
54
|
+
umountSnapshot();
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('cross-thread SharedArrayBuffer', async () => {
|
|
58
|
+
const buffer = new SharedArrayBuffer(0x100000);
|
|
59
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
60
|
+
|
|
61
|
+
assert.equal(await runWorker({ buffer }), 'continue');
|
|
62
|
+
assert(fs.existsSync(path + '/worker-file.ts'));
|
|
63
|
+
|
|
64
|
+
umount();
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('concurrent writers allocate distinct IDs', async () => {
|
|
68
|
+
const buffer = new SharedArrayBuffer(0x1000000);
|
|
69
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
70
|
+
|
|
71
|
+
const count = 50,
|
|
72
|
+
workers = 4;
|
|
73
|
+
for (let id = 0; id < workers; id++) fs.mkdirSync(`${path}/w${id}`);
|
|
74
|
+
|
|
75
|
+
const results = await Promise.all(Array.from({ length: workers }, (_, id) => runWorker({ buffer, mode: 'write', id, count })));
|
|
76
|
+
assert.deepEqual(results, Array(workers).fill('continue'));
|
|
77
|
+
|
|
78
|
+
const [check, umountCheck] = await mountBuffer(buffer);
|
|
79
|
+
const inodes = new Set<number>();
|
|
80
|
+
for (let id = 0; id < workers; id++) {
|
|
81
|
+
for (let i = 0; i < count; i++) {
|
|
82
|
+
const file = `${check}/w${id}/${i}`;
|
|
83
|
+
assert.equal(fs.readFileSync(file, 'utf8'), `worker ${id} file ${i}`);
|
|
84
|
+
inodes.add(fs.statSync(file).ino);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
assert.equal(inodes.size, workers * count);
|
|
88
|
+
|
|
89
|
+
umount();
|
|
90
|
+
umountCheck();
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test('readers see whole versions while another thread writes', async () => {
|
|
94
|
+
const buffer = new SharedArrayBuffer(0x1000000);
|
|
95
|
+
const store = new SingleBufferStoreV2(buffer, 0, undefined, { format: true });
|
|
96
|
+
|
|
97
|
+
store.set(100, new Uint8Array(1000).fill(1));
|
|
98
|
+
const gate = new Int32Array(new SharedArrayBuffer(4));
|
|
99
|
+
const reader = runWorker({ buffer, gate, mode: 'read' });
|
|
100
|
+
Atomics.wait(gate, 0, 0, 5000);
|
|
101
|
+
|
|
102
|
+
for (let i = 0; i < 2000; i++) store.set(100, new Uint8Array(1000 + ((i * 7919) % 50000)).fill(i % 255 || 1));
|
|
103
|
+
store.set(101, new Uint8Array(1));
|
|
104
|
+
|
|
105
|
+
assert.ok(((await reader) as number) > 0, 'the reader never read anything');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test('threads creating the same file system at once share it', async () => {
|
|
109
|
+
const buffer = new SharedArrayBuffer(0x400000);
|
|
110
|
+
const uuids = await Promise.all(Array.from({ length: 4 }, () => runWorker({ buffer, mode: 'setup' })));
|
|
111
|
+
assert.equal(new Set(uuids).size, 1);
|
|
112
|
+
assert.equal(Buffer.from(new SuperBlock(buffer, 0).uuid).toString('hex'), uuids[0]);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test('partial reads and writes', () => {
|
|
116
|
+
const store = new SingleBufferStoreV2(new ArrayBuffer(0x100000), 0, undefined, { format: true });
|
|
117
|
+
store.set(10, new Uint8Array([1, 2, 3, 4]));
|
|
118
|
+
assert.deepEqual(store.get(10, 1, 3), new Uint8Array([2, 3]));
|
|
119
|
+
assert.deepEqual(store.get(10, 2), new Uint8Array([3, 4]));
|
|
120
|
+
|
|
121
|
+
store.set(10, new Uint8Array([9]), 2);
|
|
122
|
+
assert.deepEqual(store.get(10), new Uint8Array([1, 2, 9, 4]));
|
|
123
|
+
|
|
124
|
+
store.set(10, new Uint8Array([7]), 6);
|
|
125
|
+
assert.deepEqual(store.get(10), new Uint8Array([1, 2, 9, 4, 0, 0, 7]));
|
|
126
|
+
|
|
127
|
+
store.set(10, new Uint8Array([5]));
|
|
128
|
+
assert.deepEqual(store.get(10), new Uint8Array([5]));
|
|
129
|
+
|
|
130
|
+
store.set(11, new Uint8Array([8]), 2);
|
|
131
|
+
assert.deepEqual(store.get(11), new Uint8Array([0, 0, 8]));
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test('appended data is rarely moved', () => {
|
|
135
|
+
const store = new SingleBufferStoreV2(new ArrayBuffer(0x100000), 0, undefined, { format: true });
|
|
136
|
+
const offsetOf = (id: number) => store['read'](index => index.get(BigInt(id))!.offset);
|
|
137
|
+
|
|
138
|
+
store.set(10, new Uint8Array(100).fill(1));
|
|
139
|
+
let moves = 0;
|
|
140
|
+
for (let i = 0; i < 200; i++) {
|
|
141
|
+
const before = offsetOf(10);
|
|
142
|
+
store.set(10, new Uint8Array(100).fill(i + 2), 100 + i * 100);
|
|
143
|
+
if (offsetOf(10) != before) moves++;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Capacity grows by half each time data moves
|
|
147
|
+
assert.ok(moves <= Math.ceil(Math.log(201) / Math.log(1.5)), `data moved ${moves} times`);
|
|
148
|
+
|
|
149
|
+
const data = store.get(10)!;
|
|
150
|
+
assert.equal(data.length, 20100);
|
|
151
|
+
assert.ok(data.subarray(0, 100).every(byte => byte == 1));
|
|
152
|
+
assert.ok(data.subarray(20000).every(byte => byte == 201));
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test('writing past the end of a truncated file leaves zeros between', async () => {
|
|
156
|
+
const [path, umount] = await mountBuffer(new ArrayBuffer(0x100000));
|
|
157
|
+
fs.writeFileSync(path + '/file', Buffer.alloc(100, 1));
|
|
158
|
+
fs.truncateSync(path + '/file', 10);
|
|
159
|
+
|
|
160
|
+
const fd = fs.openSync(path + '/file', 'r+');
|
|
161
|
+
fs.writeSync(fd, Buffer.alloc(10, 2), 0, 10, 50);
|
|
162
|
+
fs.closeSync(fd);
|
|
163
|
+
|
|
164
|
+
const data = fs.readFileSync(path + '/file');
|
|
165
|
+
assert.equal(data.length, 60);
|
|
166
|
+
assert.ok(data.subarray(0, 10).every(byte => byte == 1));
|
|
167
|
+
assert.ok(data.subarray(10, 50).every(byte => byte == 0));
|
|
168
|
+
assert.ok(data.subarray(50).every(byte => byte == 2));
|
|
169
|
+
umount();
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test('reuses freed space', async () => {
|
|
173
|
+
const [path, umount] = await mountBuffer(new ArrayBuffer(0x100000));
|
|
174
|
+
const baseline = used(path);
|
|
175
|
+
|
|
176
|
+
// Writes far more than the buffer can hold
|
|
177
|
+
for (let i = 0; i < 200; i++) {
|
|
178
|
+
fs.writeFileSync(path + '/churn', randomBytes(0x10000));
|
|
179
|
+
fs.appendFileSync(path + '/log', randomBytes(1024));
|
|
180
|
+
if (i % 10 == 9) fs.rmSync(path + '/log');
|
|
181
|
+
}
|
|
182
|
+
fs.rmSync(path + '/churn');
|
|
183
|
+
fs.writeFileSync(path + '/settle', '');
|
|
184
|
+
fs.rmSync(path + '/settle');
|
|
185
|
+
|
|
186
|
+
assert.ok(used(path) - baseline < 0x8000, `leaked ${used(path) - baseline} bytes`);
|
|
187
|
+
umount();
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test('failed writes leave the store unchanged', () => {
|
|
191
|
+
const store = new SingleBufferStoreV2(new ArrayBuffer(0x100000), 0, undefined, { format: true });
|
|
192
|
+
store.set(10, new Uint8Array([1, 2, 3]));
|
|
193
|
+
const before = store.usage();
|
|
194
|
+
|
|
195
|
+
assert.throws(() => store.set(12, new Uint8Array(0x200000)), { code: 'ENOSPC' });
|
|
196
|
+
assert.throws(() => store.set(10, new Uint8Array(0x200000)), { code: 'ENOSPC' });
|
|
197
|
+
|
|
198
|
+
assert.deepEqual(store.get(10), new Uint8Array([1, 2, 3]));
|
|
199
|
+
assert.equal(store.get(12), undefined);
|
|
200
|
+
assert.deepEqual(store.usage(), before);
|
|
201
|
+
|
|
202
|
+
store.set(12, new Uint8Array([4]));
|
|
203
|
+
assert.deepEqual(store.get(12), new Uint8Array([4]));
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
test('recovers from a writer that stopped while holding the lock', async () => {
|
|
207
|
+
const buffer = new SharedArrayBuffer(0x100000);
|
|
208
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
209
|
+
|
|
210
|
+
const { state_offset } = new SuperBlock(buffer, 0);
|
|
211
|
+
new Mutex(buffer, state_offset, { lease: 50 }).lock();
|
|
212
|
+
|
|
213
|
+
fs.writeFileSync(path + '/after', 'recovered');
|
|
214
|
+
assert.equal(fs.readFileSync(path + '/after', 'utf8'), 'recovered');
|
|
215
|
+
umount();
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
test('upgrades version 1 file systems in place', async () => {
|
|
219
|
+
const buffer = new ArrayBuffer(0x300000);
|
|
220
|
+
const v1 = '/sbfs-v1';
|
|
221
|
+
mount(v1, await resolveMountConfig({ backend: SingleBuffer, buffer, v2: false }));
|
|
222
|
+
|
|
223
|
+
// Enough files for several metadata blocks, with growth and deletion leaking space
|
|
224
|
+
fs.mkdirSync(v1 + '/dir/nested', { recursive: true });
|
|
225
|
+
const files = new Map<string, string>();
|
|
226
|
+
const deleted: string[] = [];
|
|
227
|
+
for (let i = 0; i < 300; i++) {
|
|
228
|
+
const file = (i % 3 ? '/dir/' : i % 2 ? '/dir/nested/' : '/') + 'f' + i;
|
|
229
|
+
let content = `file ${i}\n`.repeat((i % 7) + 1);
|
|
230
|
+
fs.writeFileSync(v1 + file, content);
|
|
231
|
+
|
|
232
|
+
if (i % 10 == 0) {
|
|
233
|
+
content += 'grown\n'.repeat(20);
|
|
234
|
+
fs.writeFileSync(v1 + file, content);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (i % 25 == 0) {
|
|
238
|
+
fs.unlinkSync(v1 + file);
|
|
239
|
+
deleted.push(file);
|
|
240
|
+
} else files.set(file, content);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const stats = new Map(Array.from(files.keys(), file => [file, fs.statSync(v1 + file)]));
|
|
244
|
+
const v1Free = 0x300000 - used(v1);
|
|
245
|
+
vfs.umount(v1);
|
|
246
|
+
assert.equal(new SuperBlock(buffer, 0).version, 1);
|
|
247
|
+
|
|
248
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
249
|
+
assert.equal(new SuperBlock(buffer, 0).version, 2);
|
|
250
|
+
|
|
251
|
+
for (const [file, content] of files) {
|
|
252
|
+
assert.deepEqual(fs.statSync(path + file), stats.get(file), `stats of ${file}`);
|
|
253
|
+
assert.equal(fs.readFileSync(path + file, 'utf8'), content, `content of ${file}`);
|
|
254
|
+
}
|
|
255
|
+
for (const file of deleted) assert.ok(!fs.existsSync(path + file), `${file} should have been deleted`);
|
|
256
|
+
|
|
257
|
+
// Space leaked by version 1 and its metadata blocks are reclaimed
|
|
258
|
+
fs.writeFileSync(path + '/new', 'written after upgrading');
|
|
259
|
+
fs.rmSync(path + '/new');
|
|
260
|
+
const free = 0x300000 - used(path);
|
|
261
|
+
assert.ok(free > v1Free + 0x10000, `only ${free} bytes are free after upgrading, compared to ${v1Free}`);
|
|
262
|
+
|
|
263
|
+
const [again, umountAgain] = await mountBuffer(buffer);
|
|
264
|
+
assert.equal(fs.readFileSync(again + '/dir/f1', 'utf8'), files.get('/dir/f1'));
|
|
265
|
+
|
|
266
|
+
umount();
|
|
267
|
+
umountAgain();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test('is the default, but not used for existing version 1 file systems unless asked for', async () => {
|
|
271
|
+
const version = (buffer: ArrayBufferLike) => new SuperBlock(buffer, 0).version;
|
|
272
|
+
|
|
273
|
+
const fresh = new ArrayBuffer(0x100000);
|
|
274
|
+
await resolveMountConfig({ backend: SingleBuffer, buffer: fresh });
|
|
275
|
+
assert.equal(version(fresh), 2);
|
|
276
|
+
|
|
277
|
+
const buffer = new ArrayBuffer(0x100000);
|
|
278
|
+
await resolveMountConfig({ backend: SingleBuffer, buffer, v2: false });
|
|
279
|
+
assert.equal(version(buffer), 1);
|
|
280
|
+
|
|
281
|
+
await resolveMountConfig({ backend: SingleBuffer, buffer });
|
|
282
|
+
assert.equal(version(buffer), 1);
|
|
283
|
+
|
|
284
|
+
await resolveMountConfig({ backend: SingleBuffer, buffer, v2: true });
|
|
285
|
+
assert.equal(version(buffer), 2);
|
|
286
|
+
|
|
287
|
+
mount('/sbfs-detect', await resolveMountConfig({ backend: SingleBuffer, buffer, v2: false }));
|
|
288
|
+
fs.writeFileSync('/sbfs-detect/file', 'still version 2');
|
|
289
|
+
assert.equal(version(buffer), 2);
|
|
290
|
+
vfs.umount('/sbfs-detect');
|
|
291
|
+
|
|
292
|
+
assert.throws(() => new SingleBufferStore(buffer), { code: 'EIO' });
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
test('reliability across varied file sizes', async () => {
|
|
296
|
+
const buffer = new ArrayBuffer(0x400000);
|
|
297
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
298
|
+
|
|
299
|
+
const filePath = `${path}/payload.bin`;
|
|
300
|
+
const growthSizes = [0, 1, 17, 512, 8192, 65535, 262144, 524288];
|
|
301
|
+
const shrinkSizes = [262144, 4096, 128, 0];
|
|
302
|
+
|
|
303
|
+
const verifySnapshot = async (expected: Buffer, size: number) => {
|
|
304
|
+
const [verify, umountVerify] = await mountBuffer(buffer);
|
|
305
|
+
try {
|
|
306
|
+
const reopened = fs.readFileSync(`${verify}/payload.bin`);
|
|
307
|
+
assert.strictEqual(reopened.byteLength, size, `snapshot size mismatch for ${size} bytes`);
|
|
308
|
+
assert.deepStrictEqual(reopened, expected, `snapshot content mismatch for ${size} bytes`);
|
|
309
|
+
} finally {
|
|
310
|
+
umountVerify();
|
|
311
|
+
}
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
try {
|
|
315
|
+
for (const size of [...growthSizes, ...shrinkSizes]) {
|
|
316
|
+
const payload = size ? randomBytes(size) : Buffer.alloc(0);
|
|
317
|
+
fs.writeFileSync(filePath, payload);
|
|
318
|
+
const direct = fs.readFileSync(filePath);
|
|
319
|
+
assert.strictEqual(direct.byteLength, size, `direct size mismatch for ${size} bytes`);
|
|
320
|
+
assert.deepStrictEqual(direct, payload, `direct content mismatch for ${size} bytes`);
|
|
321
|
+
await verifySnapshot(direct, size);
|
|
322
|
+
}
|
|
323
|
+
} finally {
|
|
324
|
+
umount();
|
|
325
|
+
}
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
test('files with uneven sizes', async () => {
|
|
329
|
+
const [path, umount] = await mountBuffer(new ArrayBuffer(0x400000));
|
|
330
|
+
|
|
331
|
+
const sizes = [1, 17, 257, 3, 5, 13, 1023, 4095, 7, 9];
|
|
332
|
+
try {
|
|
333
|
+
for (let i = 0; i < 400; i++) fs.writeFileSync(`${path}/f${i}.txt`, Buffer.alloc(sizes[i % sizes.length], i & 0xff));
|
|
334
|
+
for (let i = 0; i < 400; i += 37) {
|
|
335
|
+
const expected = Buffer.alloc(sizes[i % sizes.length], i & 0xff);
|
|
336
|
+
assert.deepStrictEqual(fs.readFileSync(`${path}/f${i}.txt`), expected, `content mismatch at f${i}.txt`);
|
|
337
|
+
}
|
|
338
|
+
} finally {
|
|
339
|
+
umount();
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
});
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { writeFileSync as _write } from 'node:fs';
|
|
3
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
4
|
+
import { SingleBufferStore as SingleBufferStoreV2, SuperBlock } from '../../dist/backends/single_buffer/v2.js';
|
|
5
|
+
import { configureSingle, fs, SingleBuffer } from '../../dist/index.js';
|
|
6
|
+
import { setupLogs } from '../logs.js';
|
|
7
|
+
|
|
8
|
+
setupLogs('<worker>');
|
|
9
|
+
|
|
10
|
+
const { buffer, gate, mode = 'create', id = 0, count = 0 } = workerData;
|
|
11
|
+
|
|
12
|
+
try {
|
|
13
|
+
if (mode != 'read') await configureSingle({ backend: SingleBuffer, buffer });
|
|
14
|
+
|
|
15
|
+
switch (mode) {
|
|
16
|
+
case 'create': {
|
|
17
|
+
const content = 'console.log("this file was created by the worker")';
|
|
18
|
+
fs.writeFileSync('/worker-file.ts', content, 'utf-8');
|
|
19
|
+
assert.equal(fs.readFileSync('/worker-file.ts', 'utf-8'), content);
|
|
20
|
+
break;
|
|
21
|
+
}
|
|
22
|
+
case 'setup':
|
|
23
|
+
parentPort.postMessage(Buffer.from(new SuperBlock(buffer, 0).uuid).toString('hex'));
|
|
24
|
+
process.exit(0);
|
|
25
|
+
case 'write':
|
|
26
|
+
for (let i = 0; i < count; i++) fs.writeFileSync(`/w${id}/${i}`, `worker ${id} file ${i}`);
|
|
27
|
+
break;
|
|
28
|
+
case 'read': {
|
|
29
|
+
const store = new SingleBufferStoreV2(buffer);
|
|
30
|
+
const deadline = Date.now() + 5000;
|
|
31
|
+
let reads = 0;
|
|
32
|
+
while (!store.get(101) && Date.now() < deadline) {
|
|
33
|
+
const data = store.get(100);
|
|
34
|
+
const fill = data[0];
|
|
35
|
+
assert.ok(fill && data.every(byte => byte == fill), `read a torn version (${data.length} bytes, first ${fill})`);
|
|
36
|
+
if (!reads++) {
|
|
37
|
+
Atomics.store(gate, 0, 1);
|
|
38
|
+
Atomics.notify(gate, 0);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
parentPort.postMessage(reads);
|
|
42
|
+
process.exit(0);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
} catch (e) {
|
|
46
|
+
if (process.env.DEBUG) _write('tmp/shared.bin', new Uint8Array(buffer));
|
|
47
|
+
console.error(e);
|
|
48
|
+
parentPort.postMessage(e);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
parentPort.postMessage('continue');
|
|
@@ -5,7 +5,7 @@ import { suite, test } from 'node:test';
|
|
|
5
5
|
import { Worker } from 'worker_threads';
|
|
6
6
|
import { sizeof } from 'memium';
|
|
7
7
|
import { fs, mount, resolveMountConfig, SingleBuffer, vfs } from '@zenfs/core';
|
|
8
|
-
import { MetadataBlock, SuperBlock } from '@zenfs/core/backends/single_buffer';
|
|
8
|
+
import { MetadataBlock, SuperBlock } from '@zenfs/core/backends/single_buffer/v1';
|
|
9
9
|
import { setupLogs } from '../logs.js';
|
|
10
10
|
|
|
11
11
|
setupLogs();
|
|
@@ -14,13 +14,13 @@ await suite('SingleBuffer', () => {
|
|
|
14
14
|
test('filesystem restoration from original buffer (with same metadata)', async () => {
|
|
15
15
|
const buffer = new ArrayBuffer(0x100000);
|
|
16
16
|
|
|
17
|
-
const writable = await resolveMountConfig({ backend: SingleBuffer, buffer });
|
|
17
|
+
const writable = await resolveMountConfig({ backend: SingleBuffer, buffer, v2: false });
|
|
18
18
|
mount('/mnt', writable);
|
|
19
19
|
|
|
20
20
|
fs.writeFileSync('/mnt/example.ts', 'console.log("hello world")', 'utf-8');
|
|
21
21
|
const stats = fs.statSync('/mnt/example.ts');
|
|
22
22
|
|
|
23
|
-
const snapshot = await resolveMountConfig({ backend: SingleBuffer, buffer });
|
|
23
|
+
const snapshot = await resolveMountConfig({ backend: SingleBuffer, buffer, v2: false });
|
|
24
24
|
mount('/snapshot', snapshot);
|
|
25
25
|
|
|
26
26
|
const snapshotStats = fs.statSync('/snapshot/example.ts');
|
|
@@ -31,7 +31,7 @@ await suite('SingleBuffer', () => {
|
|
|
31
31
|
test('cross-thread SharedArrayBuffer', async () => {
|
|
32
32
|
const sharedBuffer = new SharedArrayBuffer(0x100000);
|
|
33
33
|
|
|
34
|
-
const writable = await resolveMountConfig({ backend: SingleBuffer, buffer: sharedBuffer });
|
|
34
|
+
const writable = await resolveMountConfig({ backend: SingleBuffer, buffer: sharedBuffer, v2: false });
|
|
35
35
|
fs.mkdirSync('/shared');
|
|
36
36
|
mount('/shared', writable);
|
|
37
37
|
|
|
@@ -141,7 +141,7 @@ await suite('SingleBuffer', () => {
|
|
|
141
141
|
const mountPoint = '/sbfs-reliability';
|
|
142
142
|
const verifyMountPoint = '/sbfs-verify';
|
|
143
143
|
const buffer = new ArrayBuffer(0x400000);
|
|
144
|
-
const writable = await resolveMountConfig({ backend: SingleBuffer, buffer, label: 'reliability' });
|
|
144
|
+
const writable = await resolveMountConfig({ backend: SingleBuffer, buffer, label: 'reliability', v2: false });
|
|
145
145
|
mount(mountPoint, writable);
|
|
146
146
|
|
|
147
147
|
const filePath = `${mountPoint}/payload.bin`;
|
|
@@ -186,7 +186,7 @@ await suite('SingleBuffer', () => {
|
|
|
186
186
|
test('keeps metadata aligned when files have uneven sizes #309', async () => {
|
|
187
187
|
const mountPoint = '/sbfs-rotation';
|
|
188
188
|
const buffer = new ArrayBuffer(0x400000);
|
|
189
|
-
const writable = await resolveMountConfig({ backend: SingleBuffer, buffer, label: 'rotation' });
|
|
189
|
+
const writable = await resolveMountConfig({ backend: SingleBuffer, buffer, label: 'rotation', v2: false });
|
|
190
190
|
mount(mountPoint, writable);
|
|
191
191
|
|
|
192
192
|
// Uneven writes leave used_bytes between alignment boundaries when the metadata block fills up.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
import { Async, InMemory, InMemoryStore, StoreFS } from '@zenfs/core';
|
|
3
|
+
import assert from 'node:assert/strict';
|
|
4
|
+
import { suite, test } from 'node:test';
|
|
5
|
+
|
|
6
|
+
class AsyncMemory extends Async(StoreFS) {
|
|
7
|
+
_sync = InMemory.create({ label: 'sync' });
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
suite('Async()', () => {
|
|
11
|
+
test('sync operations are applied to the sync cache once', async () => {
|
|
12
|
+
const fs = new AsyncMemory(new InMemoryStore());
|
|
13
|
+
fs.checkRootSync();
|
|
14
|
+
await fs.ready();
|
|
15
|
+
|
|
16
|
+
const calls: string[] = [];
|
|
17
|
+
for (const key of ['mkdirSync', 'rmdirSync', 'createFileSync', 'unlinkSync'] as const) {
|
|
18
|
+
const original = (fs._sync[key] as (...args: unknown[]) => unknown).bind(fs._sync);
|
|
19
|
+
(fs._sync as any)[key] = (...args: unknown[]) => {
|
|
20
|
+
calls.push(key);
|
|
21
|
+
return original(...args);
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const options = { mode: 0o755, uid: 0, gid: 0 };
|
|
26
|
+
fs.mkdirSync('/dir', options);
|
|
27
|
+
fs.rmdirSync('/dir');
|
|
28
|
+
fs.createFileSync('/file', options);
|
|
29
|
+
fs.unlinkSync('/file');
|
|
30
|
+
await new Promise(resolve => setTimeout(resolve));
|
|
31
|
+
|
|
32
|
+
assert.deepEqual(calls, ['mkdirSync', 'rmdirSync', 'createFileSync', 'unlinkSync']);
|
|
33
|
+
assert.deepEqual(await fs.readdir('/'), []);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test('async operations see queued sync operations', async () => {
|
|
37
|
+
const fs = new AsyncMemory(new InMemoryStore());
|
|
38
|
+
fs.checkRootSync();
|
|
39
|
+
await fs.ready();
|
|
40
|
+
|
|
41
|
+
fs.mkdirSync('/dir', { mode: 0o755, uid: 0, gid: 0 });
|
|
42
|
+
assert.equal((await fs.stat('/dir')).mode & 0o777, 0o755);
|
|
43
|
+
fs.rmdirSync('/dir');
|
|
44
|
+
assert.equal(await fs.exists('/dir'), false);
|
|
45
|
+
});
|
|
46
|
+
});
|