@zenfs/core 2.7.7 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/backends/index.d.ts +3 -1
- package/dist/backends/index.js +3 -1
- package/dist/backends/single_buffer/backend.d.ts +42 -0
- package/dist/backends/single_buffer/backend.js +204 -0
- package/dist/backends/{single_buffer.d.ts → single_buffer/v1.d.ts} +24 -34
- package/dist/backends/{single_buffer.js → single_buffer/v1.js} +10 -26
- package/dist/backends/single_buffer/v2.d.ts +257 -0
- package/dist/backends/single_buffer/v2.js +1283 -0
- package/dist/backends/store/fs.d.ts +1 -1
- package/dist/backends/store/fs.js +34 -15
- package/dist/backends/store/map.d.ts +6 -6
- package/dist/backends/store/map.js +6 -6
- package/dist/backends/store/store.d.ts +18 -14
- package/dist/backends/store/store.js +11 -11
- package/dist/internal/contexts.d.ts +5 -0
- package/dist/internal/contexts.js +7 -1
- package/dist/mixins/async.js +26 -7
- package/dist/node/promises.js +6 -103
- package/dist/node/sync.js +6 -33
- package/dist/vfs/async.d.ts +4 -1
- package/dist/vfs/async.js +72 -5
- package/dist/vfs/sync.d.ts +4 -1
- package/dist/vfs/sync.js +65 -4
- package/package.json +2 -2
- package/tests/backend/single-buffer-rotate.worker.js +1 -1
- package/tests/backend/single-buffer-v2.test.ts +342 -0
- package/tests/backend/single-buffer-v2.worker.js +51 -0
- package/tests/backend/single-buffer.test.ts +6 -6
- package/tests/backend/single-buffer.worker.js +1 -0
- package/tests/common/async.test.ts +11 -0
- package/tests/common/context.test.ts +30 -0
- package/tests/fs/permissions.test.ts +27 -6
- package/tests/fs/watch.test.ts +14 -0
- package/tests/perf/config.json +31 -1
- package/tests/perf/single-buffer-random.ts +38 -0
- package/tests/perf/single-buffer-read.ts +28 -0
- package/tests/perf/single-buffer-write.ts +41 -0
- package/tests/setup/single-buffer-v2.ts +10 -0
- package/tests/setup/single-buffer.ts +1 -0
package/dist/vfs/async.js
CHANGED
|
@@ -53,10 +53,10 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
53
53
|
import { rethrow, setUVMessage, UV } from 'kerium';
|
|
54
54
|
import { decodeUTF8 } from 'utilium';
|
|
55
55
|
import * as constants from '../constants.js';
|
|
56
|
-
import { contextOf } from '../internal/contexts.js';
|
|
56
|
+
import { applyUmask, contextOf } from '../internal/contexts.js';
|
|
57
57
|
import { assertRemovable, assertSearchable, hasAccess, isDirectory, isSymbolicLink } from '../internal/inode.js';
|
|
58
58
|
import { basename, dirname, join, parse, resolve as resolvePath } from '../path.js';
|
|
59
|
-
import { normalizeMode, normalizePath } from '../utils.js';
|
|
59
|
+
import { normalizeMode, normalizePath, normalizeTime } from '../utils.js';
|
|
60
60
|
import { checkAccess, resolveFullWalk } from './config.js';
|
|
61
61
|
import { Dirent, ifToDt } from './dir.js';
|
|
62
62
|
import { Handle } from './file.js';
|
|
@@ -144,7 +144,7 @@ export async function open($, path, opt) {
|
|
|
144
144
|
if (!stats) {
|
|
145
145
|
const { euid: uid, egid: gid } = contextOf($).credentials;
|
|
146
146
|
const inode = await fs.createFile(resolved, {
|
|
147
|
-
mode,
|
|
147
|
+
mode: applyUmask($, mode),
|
|
148
148
|
uid: parentStats.mode & constants.S_ISUID ? parentStats.uid : uid,
|
|
149
149
|
gid: parentStats.mode & constants.S_ISGID ? parentStats.gid : gid,
|
|
150
150
|
});
|
|
@@ -164,7 +164,7 @@ export async function open($, path, opt) {
|
|
|
164
164
|
throw UV('EACCES', $ex);
|
|
165
165
|
if (flag & constants.O_EXCL)
|
|
166
166
|
throw UV('EEXIST', $ex);
|
|
167
|
-
if (!opt.allowDirectory && isDirectory(
|
|
167
|
+
if (!opt.allowDirectory && isDirectory(stats))
|
|
168
168
|
throw UV('EISDIR', 'open', path);
|
|
169
169
|
const handle = new Handle($, path, resolved, flag, cache.ref(resolved, stats));
|
|
170
170
|
if (flag & constants.O_TRUNC)
|
|
@@ -189,7 +189,8 @@ export async function readlink(path) {
|
|
|
189
189
|
export async function mkdir(path, options = {}) {
|
|
190
190
|
const original = normalizePath.call(this, path);
|
|
191
191
|
const { euid: uid, egid: gid } = contextOf(this).credentials;
|
|
192
|
-
const {
|
|
192
|
+
const { recursive } = options;
|
|
193
|
+
const mode = applyUmask(this, options.mode ?? 0o777);
|
|
193
194
|
const $ex = { syscall: 'mkdir', path: original };
|
|
194
195
|
const { fullPath: realParent } = await resolve(this, dirname(original), false, $ex);
|
|
195
196
|
const { fs, cache, path: target } = resolveMount(join(realParent, basename(original)), this, $ex);
|
|
@@ -400,3 +401,69 @@ export async function stat(path, lstat) {
|
|
|
400
401
|
throw UV('ENOENT', extra);
|
|
401
402
|
return stats;
|
|
402
403
|
}
|
|
404
|
+
/**
|
|
405
|
+
* Opens a handle for changing metadata, which does not need read or write access to the file.
|
|
406
|
+
*/
|
|
407
|
+
async function _openMetadata($, path, syscall, noFollow) {
|
|
408
|
+
const { fullPath, cache, path: resolved, stats } = await resolve($, path, noFollow, { syscall });
|
|
409
|
+
if (!stats)
|
|
410
|
+
throw UV('ENOENT', syscall, path);
|
|
411
|
+
return new Handle($, fullPath, resolved, constants.O_RDONLY, cache.ref(resolved, stats));
|
|
412
|
+
}
|
|
413
|
+
export async function chmod(path, mode, noFollow) {
|
|
414
|
+
const env_6 = { stack: [], error: void 0, hasError: false };
|
|
415
|
+
try {
|
|
416
|
+
path = normalizePath.call(this, path);
|
|
417
|
+
const numMode = normalizeMode(mode, -1);
|
|
418
|
+
if (numMode < 0)
|
|
419
|
+
throw UV('EINVAL', 'chmod', path);
|
|
420
|
+
const handle = __addDisposableResource(env_6, await _openMetadata(this, path, 'chmod', noFollow), true);
|
|
421
|
+
await handle.chmod(numMode);
|
|
422
|
+
emitChange(this, 'change', handle.path);
|
|
423
|
+
}
|
|
424
|
+
catch (e_6) {
|
|
425
|
+
env_6.error = e_6;
|
|
426
|
+
env_6.hasError = true;
|
|
427
|
+
}
|
|
428
|
+
finally {
|
|
429
|
+
const result_1 = __disposeResources(env_6);
|
|
430
|
+
if (result_1)
|
|
431
|
+
await result_1;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
export async function chown(path, uid, gid, noFollow) {
|
|
435
|
+
const env_7 = { stack: [], error: void 0, hasError: false };
|
|
436
|
+
try {
|
|
437
|
+
path = normalizePath.call(this, path);
|
|
438
|
+
const handle = __addDisposableResource(env_7, await _openMetadata(this, path, 'chown', noFollow), true);
|
|
439
|
+
await handle.chown(uid, gid);
|
|
440
|
+
emitChange(this, 'change', handle.path);
|
|
441
|
+
}
|
|
442
|
+
catch (e_7) {
|
|
443
|
+
env_7.error = e_7;
|
|
444
|
+
env_7.hasError = true;
|
|
445
|
+
}
|
|
446
|
+
finally {
|
|
447
|
+
const result_2 = __disposeResources(env_7);
|
|
448
|
+
if (result_2)
|
|
449
|
+
await result_2;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
export async function utimes(path, atime, mtime, noFollow) {
|
|
453
|
+
const env_8 = { stack: [], error: void 0, hasError: false };
|
|
454
|
+
try {
|
|
455
|
+
path = normalizePath.call(this, path);
|
|
456
|
+
const handle = __addDisposableResource(env_8, await _openMetadata(this, path, 'utimes', noFollow), true);
|
|
457
|
+
await handle.utimes(normalizeTime(atime), normalizeTime(mtime));
|
|
458
|
+
emitChange(this, 'change', handle.path);
|
|
459
|
+
}
|
|
460
|
+
catch (e_8) {
|
|
461
|
+
env_8.error = e_8;
|
|
462
|
+
env_8.hasError = true;
|
|
463
|
+
}
|
|
464
|
+
finally {
|
|
465
|
+
const result_3 = __disposeResources(env_8);
|
|
466
|
+
if (result_3)
|
|
467
|
+
await result_3;
|
|
468
|
+
}
|
|
469
|
+
}
|
package/dist/vfs/sync.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { PathLike } from 'node:fs';
|
|
1
|
+
import type { Mode, PathLike, TimeLike } from 'node:fs';
|
|
2
2
|
import type { V_Context } from '../context.js';
|
|
3
3
|
import type { InodeLike } from '../internal/inode.js';
|
|
4
4
|
import type { MkdirOptions, OpenOptions, ReaddirOptions, ResolvedPath } from './shared.js';
|
|
@@ -23,3 +23,6 @@ export declare function readdir(this: V_Context, path: PathLike, options?: Readd
|
|
|
23
23
|
export declare function rename(this: V_Context, oldPath: PathLike, newPath: PathLike): void;
|
|
24
24
|
export declare function link(this: V_Context, target: PathLike, link: PathLike): void;
|
|
25
25
|
export declare function stat(this: V_Context, path: PathLike, lstat: boolean): InodeLike;
|
|
26
|
+
export declare function chmod(this: V_Context, path: PathLike, mode: Mode, noFollow?: boolean): void;
|
|
27
|
+
export declare function chown(this: V_Context, path: PathLike, uid: number, gid: number, noFollow?: boolean): void;
|
|
28
|
+
export declare function utimes(this: V_Context, path: PathLike, atime: TimeLike, mtime: TimeLike, noFollow?: boolean): void;
|
package/dist/vfs/sync.js
CHANGED
|
@@ -53,11 +53,11 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
53
53
|
import { setUVMessage, UV } from 'kerium';
|
|
54
54
|
import { decodeUTF8 } from 'utilium';
|
|
55
55
|
import * as constants from '../constants.js';
|
|
56
|
-
import { contextOf } from '../internal/contexts.js';
|
|
56
|
+
import { applyUmask, contextOf } from '../internal/contexts.js';
|
|
57
57
|
import { wrap } from '../internal/error.js';
|
|
58
58
|
import { assertRemovable, assertSearchable, hasAccess, isDirectory, isSymbolicLink } from '../internal/inode.js';
|
|
59
59
|
import { basename, dirname, join, parse, resolve as resolvePath } from '../path.js';
|
|
60
|
-
import { normalizeMode, normalizePath } from '../utils.js';
|
|
60
|
+
import { normalizeMode, normalizePath, normalizeTime } from '../utils.js';
|
|
61
61
|
import { cacheOf } from './vcache.js';
|
|
62
62
|
import { checkAccess, resolveFullWalk } from './config.js';
|
|
63
63
|
import { Dirent, ifToDt } from './dir.js';
|
|
@@ -156,7 +156,7 @@ export function open($, path, opt) {
|
|
|
156
156
|
const _ = __addDisposableResource(env_1, cache.lockSync(parentPath, 'rw', parentStats), false);
|
|
157
157
|
const { euid: uid, egid: gid } = contextOf($).credentials;
|
|
158
158
|
const inode = fs.createFileSync(resolved, {
|
|
159
|
-
mode,
|
|
159
|
+
mode: applyUmask($, mode),
|
|
160
160
|
uid: parentStats.mode & constants.S_ISUID ? parentStats.uid : uid,
|
|
161
161
|
gid: parentStats.mode & constants.S_ISGID ? parentStats.gid : gid,
|
|
162
162
|
});
|
|
@@ -205,7 +205,8 @@ export function mkdir(path, options = {}) {
|
|
|
205
205
|
const { fs, cache, path: target } = resolveMount(join(realParent, basename(original)), this, $ex);
|
|
206
206
|
const followed = realParent != dirname(original);
|
|
207
207
|
const { euid: uid, egid: gid } = contextOf(this).credentials;
|
|
208
|
-
const {
|
|
208
|
+
const { recursive } = options;
|
|
209
|
+
const mode = applyUmask(this, options.mode ?? 0o777);
|
|
209
210
|
let firstCreated;
|
|
210
211
|
const __create = (path, resolved) => {
|
|
211
212
|
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
@@ -427,3 +428,63 @@ export function stat(path, lstat) {
|
|
|
427
428
|
throw UV('ENOENT', extra);
|
|
428
429
|
return stats;
|
|
429
430
|
}
|
|
431
|
+
/**
|
|
432
|
+
* Opens a handle for changing metadata, which does not need read or write access to the file.
|
|
433
|
+
*/
|
|
434
|
+
function _openMetadata($, path, syscall, noFollow) {
|
|
435
|
+
const { fullPath, cache, path: resolved, stats } = resolve($, path, noFollow, { syscall });
|
|
436
|
+
if (!stats)
|
|
437
|
+
throw UV('ENOENT', syscall, path);
|
|
438
|
+
return new Handle($, fullPath, resolved, constants.O_RDONLY, cache.ref(resolved, stats));
|
|
439
|
+
}
|
|
440
|
+
export function chmod(path, mode, noFollow) {
|
|
441
|
+
const env_6 = { stack: [], error: void 0, hasError: false };
|
|
442
|
+
try {
|
|
443
|
+
path = normalizePath.call(this, path);
|
|
444
|
+
const numMode = normalizeMode(mode, -1);
|
|
445
|
+
if (numMode < 0)
|
|
446
|
+
throw UV('EINVAL', 'chmod', path);
|
|
447
|
+
const handle = __addDisposableResource(env_6, _openMetadata(this, path, 'chmod', noFollow), false);
|
|
448
|
+
handle.chmodSync(numMode);
|
|
449
|
+
emitChange(this, 'change', handle.path);
|
|
450
|
+
}
|
|
451
|
+
catch (e_6) {
|
|
452
|
+
env_6.error = e_6;
|
|
453
|
+
env_6.hasError = true;
|
|
454
|
+
}
|
|
455
|
+
finally {
|
|
456
|
+
__disposeResources(env_6);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
export function chown(path, uid, gid, noFollow) {
|
|
460
|
+
const env_7 = { stack: [], error: void 0, hasError: false };
|
|
461
|
+
try {
|
|
462
|
+
path = normalizePath.call(this, path);
|
|
463
|
+
const handle = __addDisposableResource(env_7, _openMetadata(this, path, 'chown', noFollow), false);
|
|
464
|
+
handle.chownSync(uid, gid);
|
|
465
|
+
emitChange(this, 'change', handle.path);
|
|
466
|
+
}
|
|
467
|
+
catch (e_7) {
|
|
468
|
+
env_7.error = e_7;
|
|
469
|
+
env_7.hasError = true;
|
|
470
|
+
}
|
|
471
|
+
finally {
|
|
472
|
+
__disposeResources(env_7);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
export function utimes(path, atime, mtime, noFollow) {
|
|
476
|
+
const env_8 = { stack: [], error: void 0, hasError: false };
|
|
477
|
+
try {
|
|
478
|
+
path = normalizePath.call(this, path);
|
|
479
|
+
const handle = __addDisposableResource(env_8, _openMetadata(this, path, 'utimes', noFollow), false);
|
|
480
|
+
handle.utimesSync(normalizeTime(atime), normalizeTime(mtime));
|
|
481
|
+
emitChange(this, 'change', handle.path);
|
|
482
|
+
}
|
|
483
|
+
catch (e_8) {
|
|
484
|
+
env_8.error = e_8;
|
|
485
|
+
env_8.hasError = true;
|
|
486
|
+
}
|
|
487
|
+
finally {
|
|
488
|
+
__disposeResources(env_8);
|
|
489
|
+
}
|
|
490
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zenfs/core",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.8.0",
|
|
4
4
|
"description": "A filesystem, anywhere",
|
|
5
5
|
"funding": {
|
|
6
6
|
"type": "individual",
|
|
@@ -79,7 +79,7 @@
|
|
|
79
79
|
"buffer": "^6.0.3",
|
|
80
80
|
"eventemitter3": "^5.0.1",
|
|
81
81
|
"kerium": "^1.5.0",
|
|
82
|
-
"memium": "^1.
|
|
82
|
+
"memium": "^1.1.0",
|
|
83
83
|
"utilium": "^3.7.0"
|
|
84
84
|
},
|
|
85
85
|
"devDependencies": {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
2
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
3
|
-
import { SuperBlock } from '../../dist/backends/single_buffer.js';
|
|
3
|
+
import { SuperBlock } from '../../dist/backends/single_buffer/v1.js';
|
|
4
4
|
import { setupLogs } from '../logs.js';
|
|
5
5
|
|
|
6
6
|
setupLogs('<worker>');
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
// SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
import { fs, mount, resolveMountConfig, SingleBuffer, vfs } from '@zenfs/core';
|
|
3
|
+
import { SingleBufferStore } from '@zenfs/core/backends/single_buffer/v1';
|
|
4
|
+
import { SingleBufferStore as SingleBufferStoreV2, SuperBlock } from '@zenfs/core/backends/single_buffer/v2';
|
|
5
|
+
import { Mutex } from 'memium/locks';
|
|
6
|
+
import assert from 'node:assert/strict';
|
|
7
|
+
import { randomBytes } from 'node:crypto';
|
|
8
|
+
import { suite, test } from 'node:test';
|
|
9
|
+
import { Worker } from 'worker_threads';
|
|
10
|
+
import { setupLogs } from '../logs.js';
|
|
11
|
+
|
|
12
|
+
setupLogs();
|
|
13
|
+
|
|
14
|
+
function runWorker(workerData: object): Promise<unknown> {
|
|
15
|
+
const worker = new Worker(import.meta.dirname + '/single-buffer-v2.worker.js', { workerData });
|
|
16
|
+
const { promise, resolve, reject } = Promise.withResolvers<unknown>();
|
|
17
|
+
const timeout = setTimeout(() => reject(new Error('worker timed out')), 5000);
|
|
18
|
+
worker.on('error', reject);
|
|
19
|
+
worker.on('message', message => (message instanceof Error ? reject(message) : resolve(message)));
|
|
20
|
+
return promise.finally(() => {
|
|
21
|
+
clearTimeout(timeout);
|
|
22
|
+
void worker.terminate();
|
|
23
|
+
worker.unref();
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
let mounts = 0;
|
|
28
|
+
|
|
29
|
+
/** Mounts a SingleBuffer on a unique mount point */
|
|
30
|
+
async function mountBuffer(buffer: ArrayBufferLike): Promise<[path: string, umount: () => void]> {
|
|
31
|
+
const path = `/sbfs-v2-${mounts++}`;
|
|
32
|
+
mount(path, await resolveMountConfig({ backend: SingleBuffer, buffer, v2: true }));
|
|
33
|
+
return [path, () => vfs.umount(path)];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Bytes not available for new data */
|
|
37
|
+
function used(path: string): number {
|
|
38
|
+
const { blocks, bfree, bsize } = fs.statfsSync(path);
|
|
39
|
+
return (blocks - bfree) * bsize;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
await suite('SingleBuffer v2', () => {
|
|
43
|
+
test('filesystem restoration from original buffer (with same metadata)', async () => {
|
|
44
|
+
const buffer = new ArrayBuffer(0x100000);
|
|
45
|
+
|
|
46
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
47
|
+
fs.writeFileSync(path + '/example.ts', 'console.log("hello world")', 'utf-8');
|
|
48
|
+
const stats = fs.statSync(path + '/example.ts');
|
|
49
|
+
|
|
50
|
+
const [snapshot, umountSnapshot] = await mountBuffer(buffer);
|
|
51
|
+
assert.deepEqual(fs.statSync(snapshot + '/example.ts'), stats);
|
|
52
|
+
|
|
53
|
+
umount();
|
|
54
|
+
umountSnapshot();
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('cross-thread SharedArrayBuffer', async () => {
|
|
58
|
+
const buffer = new SharedArrayBuffer(0x100000);
|
|
59
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
60
|
+
|
|
61
|
+
assert.equal(await runWorker({ buffer }), 'continue');
|
|
62
|
+
assert(fs.existsSync(path + '/worker-file.ts'));
|
|
63
|
+
|
|
64
|
+
umount();
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('concurrent writers allocate distinct IDs', async () => {
|
|
68
|
+
const buffer = new SharedArrayBuffer(0x1000000);
|
|
69
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
70
|
+
|
|
71
|
+
const count = 50,
|
|
72
|
+
workers = 4;
|
|
73
|
+
for (let id = 0; id < workers; id++) fs.mkdirSync(`${path}/w${id}`);
|
|
74
|
+
|
|
75
|
+
const results = await Promise.all(Array.from({ length: workers }, (_, id) => runWorker({ buffer, mode: 'write', id, count })));
|
|
76
|
+
assert.deepEqual(results, Array(workers).fill('continue'));
|
|
77
|
+
|
|
78
|
+
const [check, umountCheck] = await mountBuffer(buffer);
|
|
79
|
+
const inodes = new Set<number>();
|
|
80
|
+
for (let id = 0; id < workers; id++) {
|
|
81
|
+
for (let i = 0; i < count; i++) {
|
|
82
|
+
const file = `${check}/w${id}/${i}`;
|
|
83
|
+
assert.equal(fs.readFileSync(file, 'utf8'), `worker ${id} file ${i}`);
|
|
84
|
+
inodes.add(fs.statSync(file).ino);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
assert.equal(inodes.size, workers * count);
|
|
88
|
+
|
|
89
|
+
umount();
|
|
90
|
+
umountCheck();
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test('readers see whole versions while another thread writes', async () => {
|
|
94
|
+
const buffer = new SharedArrayBuffer(0x1000000);
|
|
95
|
+
const store = new SingleBufferStoreV2(buffer, 0, undefined, { format: true });
|
|
96
|
+
|
|
97
|
+
store.set(100, new Uint8Array(1000).fill(1));
|
|
98
|
+
const gate = new Int32Array(new SharedArrayBuffer(4));
|
|
99
|
+
const reader = runWorker({ buffer, gate, mode: 'read' });
|
|
100
|
+
Atomics.wait(gate, 0, 0, 5000);
|
|
101
|
+
|
|
102
|
+
for (let i = 0; i < 2000; i++) store.set(100, new Uint8Array(1000 + ((i * 7919) % 50000)).fill(i % 255 || 1));
|
|
103
|
+
store.set(101, new Uint8Array(1));
|
|
104
|
+
|
|
105
|
+
assert.ok(((await reader) as number) > 0, 'the reader never read anything');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test('threads creating the same file system at once share it', async () => {
|
|
109
|
+
const buffer = new SharedArrayBuffer(0x400000);
|
|
110
|
+
const uuids = await Promise.all(Array.from({ length: 4 }, () => runWorker({ buffer, mode: 'setup' })));
|
|
111
|
+
assert.equal(new Set(uuids).size, 1);
|
|
112
|
+
assert.equal(Buffer.from(new SuperBlock(buffer, 0).uuid).toString('hex'), uuids[0]);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test('partial reads and writes', () => {
|
|
116
|
+
const store = new SingleBufferStoreV2(new ArrayBuffer(0x100000), 0, undefined, { format: true });
|
|
117
|
+
store.set(10, new Uint8Array([1, 2, 3, 4]));
|
|
118
|
+
assert.deepEqual(store.get(10, 1, 3), new Uint8Array([2, 3]));
|
|
119
|
+
assert.deepEqual(store.get(10, 2), new Uint8Array([3, 4]));
|
|
120
|
+
|
|
121
|
+
store.set(10, new Uint8Array([9]), 2);
|
|
122
|
+
assert.deepEqual(store.get(10), new Uint8Array([1, 2, 9, 4]));
|
|
123
|
+
|
|
124
|
+
store.set(10, new Uint8Array([7]), 6);
|
|
125
|
+
assert.deepEqual(store.get(10), new Uint8Array([1, 2, 9, 4, 0, 0, 7]));
|
|
126
|
+
|
|
127
|
+
store.set(10, new Uint8Array([5]));
|
|
128
|
+
assert.deepEqual(store.get(10), new Uint8Array([5]));
|
|
129
|
+
|
|
130
|
+
store.set(11, new Uint8Array([8]), 2);
|
|
131
|
+
assert.deepEqual(store.get(11), new Uint8Array([0, 0, 8]));
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test('appended data is rarely moved', () => {
|
|
135
|
+
const store = new SingleBufferStoreV2(new ArrayBuffer(0x100000), 0, undefined, { format: true });
|
|
136
|
+
const offsetOf = (id: number) => store['read'](index => index.get(BigInt(id))!.offset);
|
|
137
|
+
|
|
138
|
+
store.set(10, new Uint8Array(100).fill(1));
|
|
139
|
+
let moves = 0;
|
|
140
|
+
for (let i = 0; i < 200; i++) {
|
|
141
|
+
const before = offsetOf(10);
|
|
142
|
+
store.set(10, new Uint8Array(100).fill(i + 2), 100 + i * 100);
|
|
143
|
+
if (offsetOf(10) != before) moves++;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Capacity grows by half each time data moves
|
|
147
|
+
assert.ok(moves <= Math.ceil(Math.log(201) / Math.log(1.5)), `data moved ${moves} times`);
|
|
148
|
+
|
|
149
|
+
const data = store.get(10)!;
|
|
150
|
+
assert.equal(data.length, 20100);
|
|
151
|
+
assert.ok(data.subarray(0, 100).every(byte => byte == 1));
|
|
152
|
+
assert.ok(data.subarray(20000).every(byte => byte == 201));
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test('writing past the end of a truncated file leaves zeros between', async () => {
|
|
156
|
+
const [path, umount] = await mountBuffer(new ArrayBuffer(0x100000));
|
|
157
|
+
fs.writeFileSync(path + '/file', Buffer.alloc(100, 1));
|
|
158
|
+
fs.truncateSync(path + '/file', 10);
|
|
159
|
+
|
|
160
|
+
const fd = fs.openSync(path + '/file', 'r+');
|
|
161
|
+
fs.writeSync(fd, Buffer.alloc(10, 2), 0, 10, 50);
|
|
162
|
+
fs.closeSync(fd);
|
|
163
|
+
|
|
164
|
+
const data = fs.readFileSync(path + '/file');
|
|
165
|
+
assert.equal(data.length, 60);
|
|
166
|
+
assert.ok(data.subarray(0, 10).every(byte => byte == 1));
|
|
167
|
+
assert.ok(data.subarray(10, 50).every(byte => byte == 0));
|
|
168
|
+
assert.ok(data.subarray(50).every(byte => byte == 2));
|
|
169
|
+
umount();
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test('reuses freed space', async () => {
|
|
173
|
+
const [path, umount] = await mountBuffer(new ArrayBuffer(0x100000));
|
|
174
|
+
const baseline = used(path);
|
|
175
|
+
|
|
176
|
+
// Writes far more than the buffer can hold
|
|
177
|
+
for (let i = 0; i < 200; i++) {
|
|
178
|
+
fs.writeFileSync(path + '/churn', randomBytes(0x10000));
|
|
179
|
+
fs.appendFileSync(path + '/log', randomBytes(1024));
|
|
180
|
+
if (i % 10 == 9) fs.rmSync(path + '/log');
|
|
181
|
+
}
|
|
182
|
+
fs.rmSync(path + '/churn');
|
|
183
|
+
fs.writeFileSync(path + '/settle', '');
|
|
184
|
+
fs.rmSync(path + '/settle');
|
|
185
|
+
|
|
186
|
+
assert.ok(used(path) - baseline < 0x8000, `leaked ${used(path) - baseline} bytes`);
|
|
187
|
+
umount();
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test('failed writes leave the store unchanged', () => {
|
|
191
|
+
const store = new SingleBufferStoreV2(new ArrayBuffer(0x100000), 0, undefined, { format: true });
|
|
192
|
+
store.set(10, new Uint8Array([1, 2, 3]));
|
|
193
|
+
const before = store.usage();
|
|
194
|
+
|
|
195
|
+
assert.throws(() => store.set(12, new Uint8Array(0x200000)), { code: 'ENOSPC' });
|
|
196
|
+
assert.throws(() => store.set(10, new Uint8Array(0x200000)), { code: 'ENOSPC' });
|
|
197
|
+
|
|
198
|
+
assert.deepEqual(store.get(10), new Uint8Array([1, 2, 3]));
|
|
199
|
+
assert.equal(store.get(12), undefined);
|
|
200
|
+
assert.deepEqual(store.usage(), before);
|
|
201
|
+
|
|
202
|
+
store.set(12, new Uint8Array([4]));
|
|
203
|
+
assert.deepEqual(store.get(12), new Uint8Array([4]));
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
test('recovers from a writer that stopped while holding the lock', async () => {
|
|
207
|
+
const buffer = new SharedArrayBuffer(0x100000);
|
|
208
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
209
|
+
|
|
210
|
+
const { state_offset } = new SuperBlock(buffer, 0);
|
|
211
|
+
new Mutex(buffer, state_offset, { lease: 50 }).lock();
|
|
212
|
+
|
|
213
|
+
fs.writeFileSync(path + '/after', 'recovered');
|
|
214
|
+
assert.equal(fs.readFileSync(path + '/after', 'utf8'), 'recovered');
|
|
215
|
+
umount();
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
test('upgrades version 1 file systems in place', async () => {
|
|
219
|
+
const buffer = new ArrayBuffer(0x300000);
|
|
220
|
+
const v1 = '/sbfs-v1';
|
|
221
|
+
mount(v1, await resolveMountConfig({ backend: SingleBuffer, buffer, v2: false }));
|
|
222
|
+
|
|
223
|
+
// Enough files for several metadata blocks, with growth and deletion leaking space
|
|
224
|
+
fs.mkdirSync(v1 + '/dir/nested', { recursive: true });
|
|
225
|
+
const files = new Map<string, string>();
|
|
226
|
+
const deleted: string[] = [];
|
|
227
|
+
for (let i = 0; i < 300; i++) {
|
|
228
|
+
const file = (i % 3 ? '/dir/' : i % 2 ? '/dir/nested/' : '/') + 'f' + i;
|
|
229
|
+
let content = `file ${i}\n`.repeat((i % 7) + 1);
|
|
230
|
+
fs.writeFileSync(v1 + file, content);
|
|
231
|
+
|
|
232
|
+
if (i % 10 == 0) {
|
|
233
|
+
content += 'grown\n'.repeat(20);
|
|
234
|
+
fs.writeFileSync(v1 + file, content);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (i % 25 == 0) {
|
|
238
|
+
fs.unlinkSync(v1 + file);
|
|
239
|
+
deleted.push(file);
|
|
240
|
+
} else files.set(file, content);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const stats = new Map(Array.from(files.keys(), file => [file, fs.statSync(v1 + file)]));
|
|
244
|
+
const v1Free = 0x300000 - used(v1);
|
|
245
|
+
vfs.umount(v1);
|
|
246
|
+
assert.equal(new SuperBlock(buffer, 0).version, 1);
|
|
247
|
+
|
|
248
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
249
|
+
assert.equal(new SuperBlock(buffer, 0).version, 2);
|
|
250
|
+
|
|
251
|
+
for (const [file, content] of files) {
|
|
252
|
+
assert.deepEqual(fs.statSync(path + file), stats.get(file), `stats of ${file}`);
|
|
253
|
+
assert.equal(fs.readFileSync(path + file, 'utf8'), content, `content of ${file}`);
|
|
254
|
+
}
|
|
255
|
+
for (const file of deleted) assert.ok(!fs.existsSync(path + file), `${file} should have been deleted`);
|
|
256
|
+
|
|
257
|
+
// Space leaked by version 1 and its metadata blocks are reclaimed
|
|
258
|
+
fs.writeFileSync(path + '/new', 'written after upgrading');
|
|
259
|
+
fs.rmSync(path + '/new');
|
|
260
|
+
const free = 0x300000 - used(path);
|
|
261
|
+
assert.ok(free > v1Free + 0x10000, `only ${free} bytes are free after upgrading, compared to ${v1Free}`);
|
|
262
|
+
|
|
263
|
+
const [again, umountAgain] = await mountBuffer(buffer);
|
|
264
|
+
assert.equal(fs.readFileSync(again + '/dir/f1', 'utf8'), files.get('/dir/f1'));
|
|
265
|
+
|
|
266
|
+
umount();
|
|
267
|
+
umountAgain();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test('is the default, but not used for existing version 1 file systems unless asked for', async () => {
|
|
271
|
+
const version = (buffer: ArrayBufferLike) => new SuperBlock(buffer, 0).version;
|
|
272
|
+
|
|
273
|
+
const fresh = new ArrayBuffer(0x100000);
|
|
274
|
+
await resolveMountConfig({ backend: SingleBuffer, buffer: fresh });
|
|
275
|
+
assert.equal(version(fresh), 2);
|
|
276
|
+
|
|
277
|
+
const buffer = new ArrayBuffer(0x100000);
|
|
278
|
+
await resolveMountConfig({ backend: SingleBuffer, buffer, v2: false });
|
|
279
|
+
assert.equal(version(buffer), 1);
|
|
280
|
+
|
|
281
|
+
await resolveMountConfig({ backend: SingleBuffer, buffer });
|
|
282
|
+
assert.equal(version(buffer), 1);
|
|
283
|
+
|
|
284
|
+
await resolveMountConfig({ backend: SingleBuffer, buffer, v2: true });
|
|
285
|
+
assert.equal(version(buffer), 2);
|
|
286
|
+
|
|
287
|
+
mount('/sbfs-detect', await resolveMountConfig({ backend: SingleBuffer, buffer, v2: false }));
|
|
288
|
+
fs.writeFileSync('/sbfs-detect/file', 'still version 2');
|
|
289
|
+
assert.equal(version(buffer), 2);
|
|
290
|
+
vfs.umount('/sbfs-detect');
|
|
291
|
+
|
|
292
|
+
assert.throws(() => new SingleBufferStore(buffer), { code: 'EIO' });
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
test('reliability across varied file sizes', async () => {
|
|
296
|
+
const buffer = new ArrayBuffer(0x400000);
|
|
297
|
+
const [path, umount] = await mountBuffer(buffer);
|
|
298
|
+
|
|
299
|
+
const filePath = `${path}/payload.bin`;
|
|
300
|
+
const growthSizes = [0, 1, 17, 512, 8192, 65535, 262144, 524288];
|
|
301
|
+
const shrinkSizes = [262144, 4096, 128, 0];
|
|
302
|
+
|
|
303
|
+
const verifySnapshot = async (expected: Buffer, size: number) => {
|
|
304
|
+
const [verify, umountVerify] = await mountBuffer(buffer);
|
|
305
|
+
try {
|
|
306
|
+
const reopened = fs.readFileSync(`${verify}/payload.bin`);
|
|
307
|
+
assert.strictEqual(reopened.byteLength, size, `snapshot size mismatch for ${size} bytes`);
|
|
308
|
+
assert.deepStrictEqual(reopened, expected, `snapshot content mismatch for ${size} bytes`);
|
|
309
|
+
} finally {
|
|
310
|
+
umountVerify();
|
|
311
|
+
}
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
try {
|
|
315
|
+
for (const size of [...growthSizes, ...shrinkSizes]) {
|
|
316
|
+
const payload = size ? randomBytes(size) : Buffer.alloc(0);
|
|
317
|
+
fs.writeFileSync(filePath, payload);
|
|
318
|
+
const direct = fs.readFileSync(filePath);
|
|
319
|
+
assert.strictEqual(direct.byteLength, size, `direct size mismatch for ${size} bytes`);
|
|
320
|
+
assert.deepStrictEqual(direct, payload, `direct content mismatch for ${size} bytes`);
|
|
321
|
+
await verifySnapshot(direct, size);
|
|
322
|
+
}
|
|
323
|
+
} finally {
|
|
324
|
+
umount();
|
|
325
|
+
}
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
test('files with uneven sizes', async () => {
|
|
329
|
+
const [path, umount] = await mountBuffer(new ArrayBuffer(0x400000));
|
|
330
|
+
|
|
331
|
+
const sizes = [1, 17, 257, 3, 5, 13, 1023, 4095, 7, 9];
|
|
332
|
+
try {
|
|
333
|
+
for (let i = 0; i < 400; i++) fs.writeFileSync(`${path}/f${i}.txt`, Buffer.alloc(sizes[i % sizes.length], i & 0xff));
|
|
334
|
+
for (let i = 0; i < 400; i += 37) {
|
|
335
|
+
const expected = Buffer.alloc(sizes[i % sizes.length], i & 0xff);
|
|
336
|
+
assert.deepStrictEqual(fs.readFileSync(`${path}/f${i}.txt`), expected, `content mismatch at f${i}.txt`);
|
|
337
|
+
}
|
|
338
|
+
} finally {
|
|
339
|
+
umount();
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
});
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { writeFileSync as _write } from 'node:fs';
|
|
3
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
4
|
+
import { SingleBufferStore as SingleBufferStoreV2, SuperBlock } from '../../dist/backends/single_buffer/v2.js';
|
|
5
|
+
import { configureSingle, fs, SingleBuffer } from '../../dist/index.js';
|
|
6
|
+
import { setupLogs } from '../logs.js';
|
|
7
|
+
|
|
8
|
+
setupLogs('<worker>');
|
|
9
|
+
|
|
10
|
+
const { buffer, gate, mode = 'create', id = 0, count = 0 } = workerData;
|
|
11
|
+
|
|
12
|
+
try {
|
|
13
|
+
if (mode != 'read') await configureSingle({ backend: SingleBuffer, buffer });
|
|
14
|
+
|
|
15
|
+
switch (mode) {
|
|
16
|
+
case 'create': {
|
|
17
|
+
const content = 'console.log("this file was created by the worker")';
|
|
18
|
+
fs.writeFileSync('/worker-file.ts', content, 'utf-8');
|
|
19
|
+
assert.equal(fs.readFileSync('/worker-file.ts', 'utf-8'), content);
|
|
20
|
+
break;
|
|
21
|
+
}
|
|
22
|
+
case 'setup':
|
|
23
|
+
parentPort.postMessage(Buffer.from(new SuperBlock(buffer, 0).uuid).toString('hex'));
|
|
24
|
+
process.exit(0);
|
|
25
|
+
case 'write':
|
|
26
|
+
for (let i = 0; i < count; i++) fs.writeFileSync(`/w${id}/${i}`, `worker ${id} file ${i}`);
|
|
27
|
+
break;
|
|
28
|
+
case 'read': {
|
|
29
|
+
const store = new SingleBufferStoreV2(buffer);
|
|
30
|
+
const deadline = Date.now() + 5000;
|
|
31
|
+
let reads = 0;
|
|
32
|
+
while (!store.get(101) && Date.now() < deadline) {
|
|
33
|
+
const data = store.get(100);
|
|
34
|
+
const fill = data[0];
|
|
35
|
+
assert.ok(fill && data.every(byte => byte == fill), `read a torn version (${data.length} bytes, first ${fill})`);
|
|
36
|
+
if (!reads++) {
|
|
37
|
+
Atomics.store(gate, 0, 1);
|
|
38
|
+
Atomics.notify(gate, 0);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
parentPort.postMessage(reads);
|
|
42
|
+
process.exit(0);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
} catch (e) {
|
|
46
|
+
if (process.env.DEBUG) _write('tmp/shared.bin', new Uint8Array(buffer));
|
|
47
|
+
console.error(e);
|
|
48
|
+
parentPort.postMessage(e);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
parentPort.postMessage('continue');
|