@zenfs/core 2.6.4 → 2.6.6

package/dist/backends/cow.js

@@ -11,7 +11,8 @@ const journalOperations = new Set(['delete']);
 function isJournalOp(op) {
     return journalOperations.has(op);
 }
-const maxOpLength = Math.max(...[...journalOperations].map(op => op.length));
+// @todo use `Iterator` helper #296
+const maxOpLength = [...journalOperations].reduce((max, op) => Math.max(max, op.length), 0);
 const journalMagicString = '#journal@v0\n';
 /**
  * Tracks various operations for the CoW backend
@@ -116,11 +117,19 @@ export class CopyOnWriteFS extends FileSystem {
     _nextIno = 1;
     /** Moves the public inode numbers for a rename, including a directory's descendants */
     renameInos(oldPath, newPath) {
-        for (const [path, ino] of [...this.inos]) {
+        const changes = [];
+        for (const [path, ino] of this.inos) {
             if (path != oldPath && !path.startsWith(oldPath + '/'))
                 continue;
-            this.inos.delete(path);
-            this.inos.set(newPath + path.slice(oldPath.length), ino);
+            changes.push({
+                path,
+                add: newPath + path.slice(oldPath.length),
+                ino,
+            });
+        }
+        for (const change of changes) {
+            this.inos.delete(change.path);
+            this.inos.set(change.add, change.ino);
         }
     }
     /** Replaces the layer's inode number with the public one for `path` */
@@ -442,7 +451,7 @@ export class CopyOnWriteFS extends FileSystem {
         if (size)
             this.readable.readSync(path, data, 0, size);
         this.writable.createFileSync(path, stats);
-        this.writable.touchSync(path, stats);
+        this.writable.touchSync(path, { ...pick(stats, _inode_fields), attributes: stats.attributes, size });
         if (size)
             this.writable.writeSync(path, data, 0);
     }
@@ -460,7 +469,7 @@ export class CopyOnWriteFS extends FileSystem {
         if (size)
             await this.readable.read(path, data, 0, size);
         await this.writable.createFile(path, stats);
-        await this.writable.touch(path, stats);
+        await this.writable.touch(path, { ...pick(stats, _inode_fields), attributes: stats.attributes, size });
         if (size)
             await this.writable.write(path, data, 0);
     }

package/dist/backends/fetch.d.ts

@@ -1,6 +1,7 @@
 import type { IndexData } from '../internal/file_index.js';
 import { Index } from '../internal/file_index.js';
 import { IndexFS } from '../internal/index_fs.js';
+import type { Inode, InodeLike } from '../internal/inode.js';
 import type { SharedConfig } from './backend.js';
 /**
  * Configuration options for FetchFS.
@@ -45,6 +46,9 @@ export declare class FetchFS extends IndexFS {
      */
     protected _async(p: Promise<unknown>): void;
     constructor(index: Index, baseUrl: string, requestInit?: RequestInit, remoteWrite?: boolean | undefined);
+    protected resize(path: string, inode: Inode, oldSize: number): Promise<void> | void;
+    touch(path: string, metadata: InodeLike): Promise<void>;
+    touchSync(path: string, metadata: InodeLike): void;
     protected remove(path: string): Promise<void>;
     protected removeSync(path: string): void;
     read(path: string, buffer: Uint8Array, offset: number | undefined, end: number): Promise<void>;

package/dist/backends/fetch.js

@@ -1,10 +1,12 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 import { withErrno } from 'kerium';
 import { debug, err, warn } from 'kerium/log';
-import { decodeUTF8 } from 'utilium';
+import { _throw, decodeUTF8 } from 'utilium';
+import { Resource } from 'utilium/cache';
 import * as requests from 'utilium/requests';
 import { Index } from '../internal/file_index.js';
 import { IndexFS } from '../internal/index_fs.js';
+import { isDirectory } from '../internal/inode.js';
 import { normalizePath } from '../utils.js';
 import { S_IFREG } from '../constants.js';
 /** Parse and throw */
@@ -48,6 +50,33 @@ export class FetchFS extends IndexFS {
         this.requestInit = requestInit;
         this.remoteWrite = remoteWrite;
     }
+    resize(path, inode, oldSize) {
+        if (isDirectory(inode) || inode.size == oldSize)
+            return;
+        const url = this.baseUrl + path;
+        const resource = requests.resourcesCache.get(url) ?? new Resource(url, oldSize, {}, requests.resourcesCache);
+        resource.size = inode.size;
+        if (inode.size < oldSize)
+            return;
+        const zeroes = new Uint8Array(inode.size - oldSize);
+        resource.add(zeroes, oldSize);
+        if (this.remoteWrite)
+            return requests.set(url, zeroes, { offset: oldSize, warn }, this.requestInit).catch(parseError);
+    }
+    async touch(path, metadata) {
+        const inode = this.index.get(path) ?? _throw(withErrno('ENOENT'));
+        const oldSize = inode.size;
+        inode.update(metadata);
+        await this.resize(path, inode, oldSize);
+    }
+    touchSync(path, metadata) {
+        const inode = this.index.get(path) ?? _throw(withErrno('ENOENT'));
+        const oldSize = inode.size;
+        inode.update(metadata);
+        const pending = this.resize(path, inode, oldSize);
+        if (pending)
+            this._async(pending);
+    }
     async remove(path) {
         await requests.remove(this.baseUrl + path, { warn, cacheOnly: !this.remoteWrite }, this.requestInit);
     }

package/dist/backends/store/fs.js

@@ -418,6 +418,12 @@ export class StoreFS extends FileSystem {
         try {
             const tx = __addDisposableResource(env_9, this.transaction(), true);
             const inode = await this.findInode(tx, path);
+            if (!isDirectory(inode) && metadata.size !== undefined && metadata.size !== inode.size) {
+                const data = (await tx.get(inode.data)) ?? new Uint8Array();
+                if (metadata.size < data.byteLength)
+                    data.fill(0, metadata.size);
+                await tx.set(inode.data, metadata.size < data.byteLength ? data.subarray(0, metadata.size) : extendBuffer(data, metadata.size));
+            }
             if (inode.update(metadata)) {
                 this._add(inode.ino, path);
                 tx.setSync(inode.ino, inode);
@@ -439,6 +445,12 @@ export class StoreFS extends FileSystem {
         try {
             const tx = __addDisposableResource(env_10, this.transaction(), false);
             const inode = this.findInodeSync(tx, path);
+            if (!isDirectory(inode) && metadata.size !== undefined && metadata.size !== inode.size) {
+                const data = tx.getSync(inode.data) ?? new Uint8Array();
+                if (metadata.size < data.byteLength)
+                    data.fill(0, metadata.size);
+                tx.setSync(inode.data, metadata.size < data.byteLength ? data.subarray(0, metadata.size) : extendBuffer(data, metadata.size));
+            }
             if (inode.update(metadata)) {
                 this._add(inode.ino, path);
                 tx.setSync(inode.ino, inode);
@@ -921,7 +933,8 @@ export class StoreFS extends FileSystem {
     _lastID;
     /** Allocates a new ID and adds the ID/path */
     allocNew(path) {
-        this._lastID ??= Math.max(...this._paths.keys());
+        // @todo use `Iterator` helper #296
+        this._lastID ||= Array.from(this._paths.keys()).reduce((max, ino) => Math.max(max, ino), 0);
         this._lastID += 2;
         const id = this._lastID;
         if (id > size_max)

package/dist/internal/file_index.d.ts

@@ -17,6 +17,7 @@ export declare const version = 1;
  * @internal
  */
 export declare class Index extends Map<string, Inode> {
+    #private;
     maxSize: number;
     /**
      * Converts the index to JSON
@@ -26,6 +27,7 @@ export declare class Index extends Map<string, Inode> {
      * Converts the index to a string
      */
     toString(): string;
+    clear(): void;
     /**
      * Get the size in bytes of the index (including the size reported for each entry)
      */

package/dist/internal/file_index.js

@@ -21,7 +21,8 @@ export class Index extends Map {
         return {
             version,
             maxSize: this.maxSize,
-            entries: Object.fromEntries([...this].map(([k, v]) => [k, v.toJSON()])),
+            // @todo use `Iterator.map` #296
+            entries: Object.fromEntries(Array.from(this).map(([k, v]) => [k, v.toJSON()])),
         };
     }
     /**
@@ -30,6 +31,10 @@ export class Index extends Map {
     toString() {
         return JSON.stringify(this.toJSON());
     }
+    clear() {
+        super.clear();
+        this.#lastAlloc = 0;
+    }
     /**
      * Get the size in bytes of the index (including the size reported for each entry)
      */
@@ -74,12 +79,16 @@ export class Index extends Map {
         }
         return entries;
     }
+    #lastAlloc = 0;
     /**
      * Get the next available ID in the index
      * @internal
      */
     _alloc() {
-        return Math.max(...[...this.values()].flatMap(i => [i.ino, i.data])) + 1;
+        // @todo use `Iterator.flatMap` #296
+        this.#lastAlloc ||= Array.from(this.values()).reduce((max, inode) => Math.max(max, inode.ino, inode.data), 0);
+        this.#lastAlloc++;
+        return this.#lastAlloc++;
     }
     /**
      * Gets a list of entries for each directory in the index.

package/dist/polyfills.js

@@ -43,7 +43,10 @@ Uint8Array.fromBase64 ??=
 Uint8Array.prototype.toHex ??=
     (warn('Using a polyfill of Uint8Array.prototype.toHex'),
         function toHex() {
-            return [...this].map(b => b.toString(16).padStart(2, '0')).join('');
+            // @todo use `Iterator.map` #296
+            return Array.from(this)
+                .map(b => b.toString(16).padStart(2, '0'))
+                .join('');
         });
 Uint8Array.fromHex ??=
     (warn('Using a polyfill of Uint8Array.fromHex'),

package/dist/vfs/file.js

@@ -675,7 +675,8 @@ export { Handle };
  */
 export function toFD(file) {
     const map = contextOf(file.context).descriptors;
-    const fd = Math.max(map.size ? Math.max(...map.keys()) + 1 : 0, 4);
+    // @todo use `Iterator` helper #296
+    const fd = Array.from(map.keys()).reduce((max, k) => Math.max(max, k), 4);
     map.set(fd, file);
     return fd;
 }

package/dist/vfs/vcache.js

@@ -50,6 +50,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
     var e = new Error(message);
     return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
 });
+import { err } from 'kerium/log';
 import { VNode } from './vnode.js';
 /**
  * The VFS-level cache of vnodes for a single file system. Modeled after Linux's dcache/icache.
@@ -76,6 +77,8 @@ export class VCache {
      */
     ref(path, inode) {
         let node = this.byIno.get(inode.ino);
+        if (node && node.inode.nlink === node.paths.size && !node.paths.has(path))
+            err(`vcache.ref: vnode for ${this.fs.label || this.fs.uuid}:${inode.ino} has an nlink of ${node.inode.nlink} but referenced by more paths [#314]`);
         if (!node) {
             node = new VNode(this.fs, path, inode);
             this.byIno.set(inode.ino, node);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@zenfs/core",
-	"version": "2.6.4",
+	"version": "2.6.6",
 	"description": "A filesystem, anywhere",
 	"funding": {
 		"type": "individual",
@@ -90,12 +90,8 @@
 		"eslint": "^10.1.0",
 		"globals": "^17.3.0",
 		"prettier": "^3.2.5",
-		"tsx": "^4.19.1",
 		"typedoc": "^0.28.18",
 		"typescript": "^6.0.0",
 		"typescript-eslint": "^8.58.0"
-	},
-	"allowScripts": {
-		"esbuild": true
 	}
 }

package/scripts/make-index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { lstatSync, readdirSync, writeFileSync, constants as c } from 'node:fs';
+import { constants as c, lstatSync, readdirSync, writeFileSync } from 'node:fs';
 import { join, matchesGlob, relative, resolve } from 'node:path/posix';
 import { parseArgs, styleText } from 'node:util';
 
@@ -9,6 +9,7 @@ const { values: options, positionals } = parseArgs({
 		ignore: { short: 'i', type: 'string', multiple: true, default: [] },
 		output: { short: 'o', type: 'string', default: 'index.json' },
 		quiet: { short: 'q', type: 'boolean', default: false },
+		xdev: { type: 'boolean', default: false },
 		verbose: { type: 'boolean', default: false },
 	},
 	allowPositionals: true,
@@ -22,11 +23,12 @@ if (options.help) {
 path: The path to create a listing for
 
 options:
-	--help, -h		Outputs this help message
-	--quiet, -q		The command will not generate any output, including error messages.
-	--verbose		Output verbose messages
-	--output, -o <path>	Path to the output file. Defaults to listing.
-	--ignore, -i <pattern>	Ignores files which match the glob <pattern>. Can be passed multiple times.
+    -h, --help              Outputs this help message
+    -q, --quiet             The command will not generate any output, including error messages.
+        --verbose           Output verbose messages
+        --xdev              Force crossing device/FS boundaries
+    -o, --output <path>     Path to the output file. Defaults to listing.
+    -i, --ignore <pattern>  Ignores files which match the glob <pattern>. Can be passed multiple times.
 	`);
 	process.exit();
 }
@@ -75,6 +77,8 @@ function getTypeText(/** @type {{ mode: number }} */ stats) {
 	}
 }
 
+const seenDevs = new Set();
+
 /**
  * @param {string} path
  */
@@ -86,6 +90,17 @@ function computeEntries(path) {
 		}
 
 		const stats = lstatSync(path);
+		if (!seenDevs.size) seenDevs.add(stats.dev);
+		else if (!seenDevs.has(stats.dev))
+			if (options.xdev) {
+				console.warn(
+					`${styleText('yellowBright', `--xdev: Adding entries from device ${stats.dev} (${path}). You may get duplicate inos which MUST be de-duplicated manually.`)}`
+				);
+				seenDevs.add(stats.dev);
+			} else {
+				console.warn(`${styleText('yellowBright', `Ignoring ${path} because it crosses a device boundary`)}`);
+				return;
+			}
 
 		const type = getTypeText(stats);
 
@@ -97,7 +112,7 @@ function computeEntries(path) {
 		}
 	} catch (/** @type {any} */ e) {
 		if (!options.quiet) {
-			console.log(`${styleText('red', 'fail')} ${path}: ${e.message}`);
+			console.log(`${styleText('red', 'fail')} ${path}: ${e instanceof Error ? e.message : String(e)}`);
 		}
 	}
 }

package/tests/backend/cow.test.ts

@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import { CopyOnWriteFS, InMemoryStore, StoreFS } from '@zenfs/core';
+import assert from 'node:assert/strict';
+import { suite, test } from 'node:test';
+import { setupLogs } from '../logs.js';
+setupLogs();
+
+const size = 1 << 20;
+
+const options = { mode: 0o644, uid: 0, gid: 0 };
+
+suite('CopyOnWrite', () => {
+	test('Truncating a base-layer file does not copy up the bytes it discards #313', async () => {
+		const readable = new StoreFS(new InMemoryStore(undefined, 'ro'));
+		await readable.ready();
+		await readable.createFile('/big', options);
+		await readable.write('/big', new Uint8Array(size).fill(0x41), 0);
+
+		const writable = new StoreFS(new InMemoryStore(undefined, 'rw'));
+		await writable.ready();
+
+		const fs = new CopyOnWriteFS(readable, writable);
+		await fs.touch('/big', { ...(await fs.stat('/big')), size: 4 });
+
+		assert.equal((await fs.stat('/big')).size, 4);
+
+		await using tx = writable.transaction();
+		const data = (await tx.get((await writable.stat('/big')).data))!;
+
+		assert.equal(data.byteLength, 4);
+		assert.equal(data.buffer.byteLength, 4);
+	});
+});

package/tests/backend/fetch.test.ts

@@ -68,6 +68,14 @@ suite('CopyOnWrite over uncached Fetch #301', () => {
 		// Writes go through the vnode cache, so this surfaces when the handle is flushed rather than at `write`.
 		assert.throws(() => fs.fsyncSync(fd), { code: 'EAGAIN' });
 	});
+
+	test('Contents follow the size across the layers #313', async () => {
+		await fs.promises.truncate('/cow/a1.js', 4);
+		assert.equal(await fs.promises.readFile('/cow/a1.js', 'utf8'), '// C');
+
+		await fs.promises.truncate('/cow/a1.js', 10);
+		assert.equal(await fs.promises.readFile('/cow/a1.js', 'utf8'), '// C\0\0\0\0\0\0');
+	});
 });
 
 after(async () => {

package/tests/backend/single-buffer.test.ts

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
-import assert from 'node:assert';
+import assert from 'node:assert/strict';
 import { randomBytes } from 'node:crypto';
 import { suite, test } from 'node:test';
 import { Worker } from 'worker_threads';

package/tests/backend/single-buffer.worker.js

@@ -1,4 +1,4 @@
-import assert from 'node:assert';
+import assert from 'node:assert/strict';
 import { writeFileSync as _write } from 'node:fs';
 import { parentPort, workerData } from 'node:worker_threads';
 import { configureSingle, fs, SingleBuffer } from '../../dist/index.js';

package/tests/common/file_index.test.ts

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import { Index, Inode } from '@zenfs/core';
+import assert from 'node:assert/strict';
+import { suite, test } from 'node:test';
+
+/** Build an index with `n` file entries, using IDs 2..2n+1 */
+function buildIndex(n: number): Index {
+	const index = new Index();
+	index.set('/', new Inode({ ino: 0, data: 1, mode: 0o40755, nlink: 1 }));
+	for (let i = 0; i < n; i++) {
+		index.set(`/f${i}`, new Inode({ ino: 2 * i + 2, data: 2 * i + 3, mode: 0o100644, nlink: 1 }));
+	}
+	return index;
+}
+
+/** Calls `fn` with `depth` extra frames on the stack */
+function atDepth<T>(depth: number, fn: () => T): T {
+	return depth > 0 ? atDepth(depth - 1, fn) : fn();
+}
+
+suite('Index', () => {
+	test('_alloc returns an unused ID', () => {
+		const index = buildIndex(10);
+		assert.equal(index._alloc(), 22);
+	});
+
+	test('_alloc does not re-use IDs', () => {
+		const index = buildIndex(10);
+		const ids = new Set<number>();
+
+		for (let i = 0; i < 100; i++) {
+			const id = index._alloc();
+			assert.equal(index.getByID(id), undefined, `\`_alloc\` returned in-use ID ${id}`);
+			assert(!ids.has(id), `\`_alloc\` returned duplicate ID ${id}`);
+			ids.add(id);
+		}
+	});
+
+	test('_alloc works with a large index #312', () => {
+		const index = buildIndex(100_000);
+		assert.equal(index._alloc(), 200_002);
+	});
+
+	test('_alloc works with a large index and a deep stack #312', () => {
+		const index = buildIndex(60_000);
+		assert.equal(
+			atDepth(1000, () => index._alloc()),
+			120_002
+		);
+	});
+});

package/tests/common/inode.test.ts

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 import { Inode } from '@zenfs/core';
-import assert from 'node:assert';
+import assert from 'node:assert/strict';
 import { suite, test } from 'node:test';
 
 suite('Inode manipulation', () => {

package/tests/fs/truncate.test.ts

@@ -34,6 +34,17 @@ suite('Truncating', config('truncate'), () => {
 		fs.closeSync(fd);
 	});
 
+	test('Contents follow the size #313', config('sync'), () => {
+		const path = 'truncate-contents.txt';
+		fs.writeFileSync(path, 'SECRETDATA');
+
+		fs.truncateSync(path, 4);
+		assert.equal(fs.readFileSync(path, 'utf8'), 'SECR');
+
+		fs.truncateSync(path, 10);
+		assert.equal(fs.readFileSync(path, 'utf8'), 'SECR\0\0\0\0\0\0');
+	});
+
 	const statSize = async (path: string) => (await fs.promises.stat(path)).size;
 
 	test('Async path functions', config('async'), async () => {
@@ -64,4 +75,15 @@ suite('Truncating', config('truncate'), () => {
 
 		await handle.close();
 	});
+
+	test('Contents follow the size (async) #313', config('async'), async () => {
+		const path = 'truncate-contents-async.txt';
+		await fs.promises.writeFile(path, 'SECRETDATA');
+
+		await fs.promises.truncate(path, 4);
+		assert.equal(await fs.promises.readFile(path, 'utf8'), 'SECR');
+
+		await fs.promises.truncate(path, 10);
+		assert.equal(await fs.promises.readFile(path, 'utf8'), 'SECR\0\0\0\0\0\0');
+	});
 });