@zenfs/core 2.6.0 → 2.6.1

package/dist/backends/cow.d.ts

@@ -127,11 +127,15 @@ export declare class CopyOnWriteFS extends FileSystem {
      * Helper function:
      * - Ensures p is on writable before proceeding. Throws an error if it doesn't exist.
      * - Calls f to perform operation on writable.
+     *
+     * `keep` is the size the file is about to be truncated to, when the caller knows it.
+     * Bytes past it are discarded by the operation that follows, so they are not worth copying up.
      */
     private copyForWriteSync;
     private copyForWrite;
     /**
      * Copy from readable to writable storage.
+     * Only `[0, keep)` is copied; the caller is responsible for setting the final size.
      * PRECONDITION: File does not exist on writable storage.
      */
     private copyToWritableSync;

package/dist/backends/cow.js

@@ -202,11 +202,11 @@ export class CopyOnWriteFS extends FileSystem {
         }
     }
     async touch(path, metadata) {
-        await this.copyForWrite(path);
+        await this.copyForWrite(path, metadata.size);
         await this.writable.touch(path, metadata);
     }
     touchSync(path, metadata) {
-        this.copyForWriteSync(path);
+        this.copyForWriteSync(path, metadata.size);
         this.writable.touchSync(path, metadata);
     }
     async createFile(path, options) {
@@ -360,32 +360,36 @@ export class CopyOnWriteFS extends FileSystem {
      * Helper function:
      * - Ensures p is on writable before proceeding. Throws an error if it doesn't exist.
      * - Calls f to perform operation on writable.
+     *
+     * `keep` is the size the file is about to be truncated to, when the caller knows it.
+     * Bytes past it are discarded by the operation that follows, so they are not worth copying up.
      */
-    copyForWriteSync(path) {
+    copyForWriteSync(path, keep) {
         if (!this.existsSync(path))
             throw withErrno('ENOENT');
         if (!this.writable.existsSync(dirname(path))) {
             this.createParentDirectoriesSync(path);
         }
         if (!this.writable.existsSync(path)) {
-            this.copyToWritableSync(path);
+            this.copyToWritableSync(path, keep);
         }
     }
-    async copyForWrite(path) {
+    async copyForWrite(path, keep) {
         if (!(await this.exists(path)))
             throw withErrno('ENOENT');
         if (!(await this.writable.exists(dirname(path)))) {
             await this.createParentDirectories(path);
         }
         if (!(await this.writable.exists(path))) {
-            return this.copyToWritable(path);
+            return this.copyToWritable(path, keep);
         }
     }
     /**
      * Copy from readable to writable storage.
+     * Only `[0, keep)` is copied; the caller is responsible for setting the final size.
      * PRECONDITION: File does not exist on writable storage.
      */
-    copyToWritableSync(path) {
+    copyToWritableSync(path, keep) {
         const stats = this.readable.statSync(path);
         if (isDirectory(stats)) {
             this.writable.mkdirSync(path, stats);
@@ -394,13 +398,16 @@ export class CopyOnWriteFS extends FileSystem {
             }
             return;
         }
-        const data = new Uint8Array(stats.size);
-        this.readable.readSync(path, data, 0, data.byteLength);
+        const size = Math.min(stats.size, keep ?? stats.size);
+        const data = new Uint8Array(size);
+        if (size)
+            this.readable.readSync(path, data, 0, size);
         this.writable.createFileSync(path, stats);
         this.writable.touchSync(path, stats);
-        this.writable.writeSync(path, data, 0);
+        if (size)
+            this.writable.writeSync(path, data, 0);
     }
-    async copyToWritable(path) {
+    async copyToWritable(path, keep) {
         const stats = await this.readable.stat(path);
         if (isDirectory(stats)) {
             await this.writable.mkdir(path, stats);
@@ -409,11 +416,14 @@ export class CopyOnWriteFS extends FileSystem {
             }
             return;
         }
-        const data = new Uint8Array(stats.size);
-        await this.readable.read(path, data, 0, stats.size);
+        const size = Math.min(stats.size, keep ?? stats.size);
+        const data = new Uint8Array(size);
+        if (size)
+            await this.readable.read(path, data, 0, size);
         await this.writable.createFile(path, stats);
         await this.writable.touch(path, stats);
-        await this.writable.write(path, data, 0);
+        if (size)
+            await this.writable.write(path, data, 0);
     }
 }
 const _CopyOnWrite = {

package/dist/backends/fetch.d.ts

@@ -39,6 +39,10 @@ export declare class FetchFS extends IndexFS {
      * @internal @hidden
      */
     _asyncDone: Promise<unknown>;
+    /**
+     * Queue an async operation started from a sync method.
+     * Failures aren't rethrown to prevent further async ops from failing
+     */
     protected _async(p: Promise<unknown>): void;
     constructor(index: Index, baseUrl: string, requestInit?: RequestInit, remoteWrite?: boolean | undefined);
     protected remove(path: string): Promise<void>;

package/dist/backends/fetch.js

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 import { withErrno } from 'kerium';
-import { err, warn } from 'kerium/log';
+import { debug, err, warn } from 'kerium/log';
 import { decodeUTF8 } from 'utilium';
 import * as requests from 'utilium/requests';
 import { Index } from '../internal/file_index.js';
@@ -35,8 +35,12 @@ export class FetchFS extends IndexFS {
      * @internal @hidden
      */
     _asyncDone = Promise.resolve();
+    /**
+     * Queue an async operation started from a sync method.
+     * Failures aren't rethrown to prevent further async ops from failing
+     */
     _async(p) {
-        this._asyncDone = this._asyncDone.then(() => p);
+        this._asyncDone = this._asyncDone.then(() => p).catch(e => debug('Async operation failed: ' + (e.message ?? e)));
     }
     constructor(index, baseUrl, requestInit = {}, remoteWrite) {
         super(0x206e6673, 'nfs', index);
@@ -74,7 +78,7 @@ export class FetchFS extends IndexFS {
         if (!data)
             throw withErrno('ENODATA');
         if (missing.length) {
-            this._async(requests.get(this.baseUrl + path, { start: offset, end, size: inode.size, warn }));
+            this._async(requests.get(this.baseUrl + path, { start: offset, end, size: inode.size, warn }, this.requestInit));
             throw withErrno('EAGAIN');
         }
         buffer.set(data);

package/dist/vfs/file.js

@@ -178,7 +178,7 @@ let Handle = (() => {
                     throw UV('EBADF', 'sync', this.path);
                 if (!this.dirty)
                     return;
-                const _ = __addDisposableResource(env_1, this.vnode.lockSync('ro'), false);
+                const _ = __addDisposableResource(env_1, this.vnode.lockSync('rw'), false);
                 this.vnode.syncSync();
             }
             catch (e_1) {
@@ -199,7 +199,7 @@ let Handle = (() => {
             if (this.closed)
                 throw UV('EBADF', 'close', this.path);
             this.syncSync();
-            this.disposeSync();
+            this.disposeSync(true);
         }
         /**
          * Cleans up. This will *not* sync the file data to the FS
@@ -385,7 +385,7 @@ let Handle = (() => {
                     throw UV('EBADF', 'sync', this.path);
                 if (!this.dirty)
                     return;
-                const _ = __addDisposableResource(env_8, await this.vnode.lock('ro'), false);
+                const _ = __addDisposableResource(env_8, await this.vnode.lock('rw'), false);
                 await this.vnode.sync();
             }
             catch (e_8) {
@@ -406,7 +406,7 @@ let Handle = (() => {
             if (this.closed)
                 throw UV('EBADF', 'close', this.path);
             await this.sync();
-            this.dispose();
+            this.dispose(true);
         }
         /**
          * Cleans up. This will *not* sync the file data to the FS
@@ -653,7 +653,7 @@ let Handle = (() => {
                 async close() {
                     const env_17 = { stack: [], error: void 0, hasError: false };
                     try {
-                        const _ = __addDisposableResource(env_17, await vnode.lock('ro'), false);
+                        const _ = __addDisposableResource(env_17, await vnode.lock('rw'), false);
                         await vnode.sync();
                     }
                     catch (e_17) {

package/dist/vfs/shared.js

@@ -52,17 +52,18 @@ export function umount(mountPoint) {
     }
     const _done = () => {
         caches.delete(fs.uuid);
-        mounts.delete(mountPoint);
-        notice('Unmounted ' + mountPoint);
+        info('Synced all changes to ' + mountPoint);
     };
     try {
         caches.get(fs.uuid)?.syncSync();
         _done();
     }
     catch {
-        warn(`Could not flush to ${mountPoint} synchronously. Unmount is being deferred.`);
+        warn(`Could not flush to ${mountPoint} synchronously.`);
         void caches.get(fs.uuid)?.sync().then(_done);
     }
+    mounts.delete(mountPoint);
+    notice('Unmounted ' + mountPoint);
 }
 /**
  * Gets the internal `FileSystem` for the path, then returns it along with the path relative to the FS' root

package/dist/vfs/vcache.js

@@ -139,7 +139,7 @@ export class VCache {
             try {
                 if (!node.dirty)
                     continue;
-                const _ = __addDisposableResource(env_1, await node.lock('ro'), false);
+                const _ = __addDisposableResource(env_1, await node.lock('rw'), false);
                 await node.sync();
                 if (!node.refs)
                     this.evict(node);
@@ -160,7 +160,7 @@ export class VCache {
             try {
                 if (!node.dirty)
                     continue;
-                const _ = __addDisposableResource(env_2, node.lockSync('ro'), false);
+                const _ = __addDisposableResource(env_2, node.lockSync('rw'), false);
                 node.syncSync();
                 if (!node.refs)
                     this.evict(node);

package/dist/vfs/vnode.d.ts

@@ -19,6 +19,8 @@ export declare class VNode extends RwLockable {
     readonly data: Resource<number>;
     /** Ranges of `data` that have been written but not yet synced to the backend */
     protected dirtyRanges: Range[];
+    /** The smallest size this vnode has been truncated to since the last sync, if it has been truncated */
+    protected truncatedTo?: number;
     /** Whether `inode` has changes that have not been synced to the backend */
     metadataDirty: boolean;
     /** The number of active references: open handles and in-progress VFS operations */
@@ -60,6 +62,10 @@ export declare class VNode extends RwLockable {
      * Note an extending truncate creates a hole, which reads as zeroes.
      */
     truncate(length: number): void;
+    /**
+     * Whether the backend has to be told about a truncation before the dirty data is written.
+     */
+    protected get needsPreTruncate(): boolean;
     /** Write all unsynced data to the backend, then metadata */
     sync(): Promise<void>;
     syncSync(): void;

package/dist/vfs/vnode.js

@@ -19,6 +19,8 @@ export class VNode extends RwLockable {
     data;
     /** Ranges of `data` that have been written but not yet synced to the backend */
     dirtyRanges = [];
+    /** The smallest size this vnode has been truncated to since the last sync, if it has been truncated */
+    truncatedTo;
     /** Whether `inode` has changes that have not been synced to the backend */
     metadataDirty = false;
     /** The number of active references: open handles and in-progress VFS operations */
@@ -151,6 +153,7 @@ export class VNode extends RwLockable {
      * Note an extending truncate creates a hole, which reads as zeroes.
      */
     truncate(length) {
+        this.truncatedTo = Math.min(this.truncatedTo ?? length, length);
         this.data.size = length;
         this.dirtyRanges = this.dirtyRanges
             .filter(range => range.start < length)
@@ -160,8 +163,17 @@ export class VNode extends RwLockable {
         this.inode.ctimeMs = Date.now();
         this.metadataDirty = true;
     }
+    /**
+     * Whether the backend has to be told about a truncation before the dirty data is written.
+     */
+    get needsPreTruncate() {
+        return !this.bypassCache && this.truncatedTo !== undefined && this.truncatedTo < this.inode.size && !this.fs.attributes.has('no_write');
+    }
     /** Write all unsynced data to the backend, then metadata */
     async sync() {
+        if (this.needsPreTruncate)
+            await this.fs.touch(this.path, { size: this.truncatedTo });
+        this.truncatedTo = undefined;
         for (const range of this.dirtyRanges) {
             for (let pos = range.start; pos < range.end;) {
                 const region = this.data.regionAt(pos);
@@ -180,6 +192,9 @@ export class VNode extends RwLockable {
         this.metadataDirty = false;
     }
     syncSync() {
+        if (this.needsPreTruncate)
+            this.fs.touchSync(this.path, { size: this.truncatedTo });
+        this.truncatedTo = undefined;
         for (const range of this.dirtyRanges) {
             for (let pos = range.start; pos < range.end;) {
                 const region = this.data.regionAt(pos);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@zenfs/core",
-	"version": "2.6.0",
+	"version": "2.6.1",
 	"description": "A filesystem, anywhere",
 	"funding": {
 		"type": "individual",

package/scripts/make-index.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
-import { readdirSync, statSync, writeFileSync } from 'node:fs';
-import { matchesGlob, relative, join, resolve } from 'node:path/posix';
+import { lstatSync, readdirSync, writeFileSync, constants as c } from 'node:fs';
+import { join, matchesGlob, relative, resolve } from 'node:path/posix';
 import { parseArgs, styleText } from 'node:util';
 
 const { values: options, positionals } = parseArgs({
@@ -47,6 +47,34 @@ const resolvedRoot = root || '.';
 
 const entries = new Map();
 
+/** @type {Record<string, import('node:util').InspectColor>} */
+const typeMap = {
+	file: 'green',
+	' dir': 'blue',
+	' sym': 'cyan',
+	' dev': 'yellow',
+	sock: 'magenta',
+	fifo: 'blue',
+};
+
+function getTypeText(/** @type {{ mode: number }} */ stats) {
+	switch (stats.mode & c.S_IFMT) {
+		case c.S_IFBLK:
+		case c.S_IFCHR:
+			return ' dev';
+		case c.S_IFIFO:
+			return 'fifo';
+		case c.S_IFSOCK:
+			return 'sock';
+		case c.S_IFREG:
+			return 'file';
+		case c.S_IFDIR:
+			return ' dir';
+		default:
+			throw `Unknown file type: 0o${(stats.mode & c.S_IFMT).toString(8)}`;
+	}
+}
+
 /**
  * @param {string} path
  */
@@ -57,22 +85,15 @@ function computeEntries(path) {
 			return;
 		}
 
-		const stats = statSync(path);
+		const stats = lstatSync(path);
 
-		if (stats.isFile()) {
-			entries.set('/' + relative(resolvedRoot, path), stats);
-			if (options.verbose) {
-				console.log(`${styleText('green', 'file')} ${path}`);
-			}
-			return;
-		}
+		const type = getTypeText(stats);
 
-		for (const file of readdirSync(path)) {
-			computeEntries(join(path, file));
-		}
 		entries.set('/' + relative(resolvedRoot, path), stats);
-		if (options.verbose) {
-			console.log(`${styleText('greenBright', ' dir')} ${path}`);
+		if (options.verbose) console.log(`${styleText(typeMap[type] || 'white', type)} ${path}`);
+
+		if (stats.isDirectory()) {
+			for (const file of readdirSync(path)) computeEntries(join(path, file));
 		}
 	} catch (/** @type {any} */ e) {
 		if (!options.quiet) {

package/tests/backend/fetch.test.ts

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
-import { Fetch, configureSingle, fs, mounts, type FetchFS } from '@zenfs/core';
+import { CopyOnWrite, Fetch, InMemory, configureSingle, fs, mounts, resolveMountConfig, type FetchFS } from '@zenfs/core';
 import assert from 'node:assert/strict';
 import { join } from 'node:path';
 import { after, suite, test } from 'node:test';
@@ -47,6 +47,29 @@ suite('Fetch with `disableAsyncCache`', () => {
 	});
 });
 
+suite('CopyOnWrite over uncached Fetch #301', () => {
+	test('Overwriting a base-layer file does not need a synchronous copy-up read', async () => {
+		const readable = await resolveMountConfig({ backend: Fetch, disableAsyncCache: true, baseUrl, index: baseUrl + indexPath });
+
+		fs.mount('/cow', await resolveMountConfig({ backend: CopyOnWrite, readable, writable: InMemory.create({}) }));
+
+		// `w+` truncates first, so none of the base file survives and nothing has to be read from it
+		fs.writeFileSync('/cow/49chars.txt', 'replaced');
+
+		assert.equal(fs.readFileSync('/cow/49chars.txt', 'utf8'), 'replaced');
+		assert.equal(fs.statSync('/cow/49chars.txt').size, 8);
+	});
+
+	test('Partially overwriting an uncached base-layer file still throws', () => {
+		const fd = fs.openSync('/cow/a.js', 'r+');
+		fs.writeSync(fd, 'x', 0);
+
+		// The bytes the write leaves alone have to be copied up, which `Fetch` can not do synchronously.
+		// Writes go through the vnode cache, so this surfaces when the handle is flushed rather than at `write`.
+		assert.throws(() => fs.fsyncSync(fd), { code: 'EAGAIN' });
+	});
+});
+
 after(async () => {
 	await (mounts.get('/') as FetchFS)._asyncDone;
 	await server.terminate();

package/tests/common.ts

@@ -21,8 +21,6 @@ const setup = await import(setupPath).catch(error => {
 // Satisfies is used to make sure that ZenFS is fully type compatible with Node.js
 export const fs = (setup.fs || defaultFS) as typeof defaultFS satisfies NodeFS;
 
-export const Stats = (setup.Stats || defaultFS.Stats) as typeof defaultFS.Stats;
-
 /**
  * A feature or function a backend may not support.
  * Tests that need one are skipped when the backend under test does not have it.

package/tests/fs/concurrency.test.ts

@@ -49,4 +49,42 @@ suite('Concurrency', config('write', 'async'), () => {
 		await a.close();
 		await b.close();
 	});
+
+	test('reads concurrent with an overwrite of the same file do not fail #303', async () => {
+		await fs.promises.writeFile('/issue-303.txt', 'original contents');
+
+		/* Every handle for a file shares one vnode, so closing any of these flushes whatever is dirty.
+		Syncing took a shared lock, letting several of these run at once and corrupt each other. */
+		for (let round = 0; round < 20; round++) {
+			const contents = `rewritten contents for round ${round}`;
+
+			const results = await Promise.allSettled([
+				...Array.from({ length: 4 }, () => fs.promises.readFile('/issue-303.txt', 'utf8')),
+				fs.promises.writeFile('/issue-303.txt', contents),
+				...Array.from({ length: 4 }, () => fs.promises.readFile('/issue-303.txt', 'utf8')),
+			]);
+
+			const rejected = results.filter(r => r.status === 'rejected');
+			assert.deepEqual(
+				rejected.map(r => r.reason.code ?? r.reason.message),
+				[]
+			);
+
+			assert.equal(await fs.promises.readFile('/issue-303.txt', 'utf8'), contents);
+		}
+	});
+
+	test('closing a handle does not fail because another handle has unsynced writes #303', async () => {
+		await fs.promises.writeFile('/issue-303-close.txt', 'original');
+
+		const reader = await fs.promises.open('/issue-303-close.txt', 'r');
+		const writer = await fs.promises.open('/issue-303-close.txt', 'r+');
+
+		// The reader wrote nothing, so its close must not report the writer's dirty data as EBUSY
+		const [closed] = await Promise.allSettled([reader.close(), writer.write(new TextEncoder().encode('DIRTY'), 0, 5, 0)]);
+		assert.equal(closed.status, 'fulfilled');
+
+		await writer.close();
+		assert.equal(await fs.promises.readFile('/issue-303-close.txt', 'utf8'), 'DIRTYnal');
+	});
 });