@zenfs/core 1.1.6 → 1.2.0

package/dist/emulation/sync.js

@@ -53,14 +53,15 @@ import { decodeUTF8, normalizeMode, normalizeOptions, normalizePath, normalizeTi
 import * as constants from './constants.js';
 import { Dir, Dirent } from './dir.js';
 import { dirname, join, parse } from './path.js';
-import { _statfs, fd2file, fdMap, file2fd, fixError, mounts, resolveMount } from './shared.js';
+import { _statfs, config, fd2file, fdMap, file2fd, fixError, mounts, resolveMount } from './shared.js';
 import { emitChange } from './watchers.js';
+import * as cache from './cache.js';
 export function renameSync(oldPath, newPath) {
     oldPath = normalizePath(oldPath);
     newPath = normalizePath(newPath);
     const oldMount = resolveMount(oldPath);
     const newMount = resolveMount(newPath);
-    if (!statSync(dirname(oldPath)).hasAccess(constants.W_OK)) {
+    if (config.checkAccess && !statSync(dirname(oldPath)).hasAccess(constants.W_OK)) {
         throw ErrnoError.With('EACCES', oldPath, 'rename');
     }
     try {
@@ -100,7 +101,7 @@ export function statSync(path, options) {
     const { fs, path: resolved } = resolveMount(realpathSync(path));
     try {
         const stats = fs.statSync(resolved);
-        if (!stats.hasAccess(constants.R_OK)) {
+        if (config.checkAccess && !stats.hasAccess(constants.R_OK)) {
             throw ErrnoError.With('EACCES', resolved, 'stat');
         }
         return options?.bigint ? new BigIntStats(stats) : stats;
@@ -145,7 +146,7 @@ export function unlinkSync(path) {
     path = normalizePath(path);
     const { fs, path: resolved } = resolveMount(path);
     try {
-        if (!fs.statSync(resolved).hasAccess(constants.W_OK)) {
+        if (config.checkAccess && !(cache.getStats(path) || fs.statSync(resolved)).hasAccess(constants.W_OK)) {
             throw ErrnoError.With('EACCES', resolved, 'unlink');
         }
         fs.unlinkSync(resolved);
@@ -174,7 +175,7 @@ function _openSync(path, _flag, _mode, resolveSymlinks = true) {
         }
         // Create the file
         const parentStats = fs.statSync(dirname(resolved));
-        if (!parentStats.hasAccess(constants.W_OK)) {
+        if (config.checkAccess && !parentStats.hasAccess(constants.W_OK)) {
             throw ErrnoError.With('EACCES', dirname(path), '_open');
         }
         if (!parentStats.isDirectory()) {
@@ -182,7 +183,7 @@ function _openSync(path, _flag, _mode, resolveSymlinks = true) {
         }
         return fs.createFileSync(resolved, flag, mode);
     }
-    if (!stats.hasAccess(mode) || !stats.hasAccess(flagToMode(flag))) {
+    if (config.checkAccess && (!stats.hasAccess(mode) || !stats.hasAccess(flagToMode(flag)))) {
         throw ErrnoError.With('EACCES', path, '_open');
     }
     if (isExclusive(flag)) {
@@ -392,7 +393,11 @@ export function rmdirSync(path) {
     path = normalizePath(path);
     const { fs, path: resolved } = resolveMount(realpathSync(path));
     try {
-        if (!fs.statSync(resolved).hasAccess(constants.W_OK)) {
+        const stats = cache.getStats(path) || fs.statSync(resolved);
+        if (!stats.isDirectory()) {
+            throw ErrnoError.With('ENOTDIR', resolved, 'rmdir');
+        }
+        if (config.checkAccess && !stats.hasAccess(constants.W_OK)) {
             throw ErrnoError.With('EACCES', resolved, 'rmdir');
         }
         fs.rmdirSync(resolved);
@@ -406,12 +411,12 @@ rmdirSync;
 export function mkdirSync(path, options) {
     options = typeof options === 'object' ? options : { mode: options };
     const mode = normalizeMode(options?.mode, 0o777);
-    path = realpathSync(normalizePath(path));
+    path = realpathSync(path);
     const { fs, path: resolved } = resolveMount(path);
     const errorPaths = { [resolved]: path };
     try {
         if (!options?.recursive) {
-            if (!fs.statSync(dirname(resolved)).hasAccess(constants.W_OK)) {
+            if (config.checkAccess && !fs.statSync(dirname(resolved)).hasAccess(constants.W_OK)) {
                 throw ErrnoError.With('EACCES', dirname(resolved), 'mkdir');
             }
             return fs.mkdirSync(resolved, mode);
@@ -422,7 +427,7 @@ export function mkdirSync(path, options) {
             errorPaths[dir] = original;
         }
         for (const dir of dirs) {
-            if (!fs.statSync(dirname(dir)).hasAccess(constants.W_OK)) {
+            if (config.checkAccess && !fs.statSync(dirname(dir)).hasAccess(constants.W_OK)) {
                 throw ErrnoError.With('EACCES', dirname(dir), 'mkdir');
             }
             fs.mkdirSync(dir, mode);
@@ -441,8 +446,13 @@ export function readdirSync(path, options) {
     const { fs, path: resolved } = resolveMount(realpathSync(path));
     let entries;
     try {
-        if (!fs.statSync(resolved).hasAccess(constants.R_OK)) {
-            throw ErrnoError.With('EACCES', path, 'readdir');
+        const stats = cache.getStats(path) || fs.statSync(resolved);
+        cache.setStats(path, stats);
+        if (config.checkAccess && !stats.hasAccess(constants.R_OK)) {
+            throw ErrnoError.With('EACCES', resolved, 'readdir');
+        }
+        if (!stats.isDirectory()) {
+            throw ErrnoError.With('ENOTDIR', resolved, 'readdir');
         }
         entries = fs.readdirSync(resolved);
     }
@@ -463,11 +473,12 @@ export function readdirSync(path, options) {
     // Iterate over entries and handle recursive case if needed
     const values = [];
     for (const entry of entries) {
-        const entryStat = fs.statSync(join(resolved, entry));
+        const entryStat = cache.getStats(join(path, entry)) || fs.statSync(join(resolved, entry));
+        cache.setStats(join(path, entry), entryStat);
         if (options?.withFileTypes) {
             values.push(new Dirent(entry, entryStat));
         }
-        else if (options?.encoding === 'buffer') {
+        else if (options?.encoding == 'buffer') {
             values.push(Buffer.from(entry));
         }
         else {
@@ -475,7 +486,7 @@ export function readdirSync(path, options) {
         }
         if (!entryStat.isDirectory() || !options?.recursive)
             continue;
-        for (const subEntry of readdirSync(join(path, entry), options)) {
+        for (const subEntry of readdirSync(join(path, entry), { ...options, _isIndirect: true })) {
             if (subEntry instanceof Dirent) {
                 subEntry.path = join(entry, subEntry.path);
                 values.push(subEntry);
@@ -488,17 +499,20 @@ export function readdirSync(path, options) {
             }
         }
     }
+    if (!options?._isIndirect) {
+        cache.clearStats();
+    }
     return values;
 }
 readdirSync;
 // SYMLINK METHODS
 export function linkSync(targetPath, linkPath) {
     targetPath = normalizePath(targetPath);
-    if (!statSync(dirname(targetPath)).hasAccess(constants.R_OK)) {
+    if (config.checkAccess && !statSync(dirname(targetPath)).hasAccess(constants.R_OK)) {
         throw ErrnoError.With('EACCES', dirname(targetPath), 'link');
     }
     linkPath = normalizePath(linkPath);
-    if (!statSync(dirname(linkPath)).hasAccess(constants.W_OK)) {
+    if (config.checkAccess && !statSync(dirname(linkPath)).hasAccess(constants.W_OK)) {
         throw ErrnoError.With('EACCES', dirname(linkPath), 'link');
     }
     const { fs, path } = resolveMount(targetPath);
@@ -507,7 +521,7 @@ export function linkSync(targetPath, linkPath) {
         throw ErrnoError.With('EXDEV', linkPath, 'link');
     }
     try {
-        if (!fs.statSync(path).hasAccess(constants.W_OK)) {
+        if (config.checkAccess && !fs.statSync(path).hasAccess(constants.W_OK)) {
             throw ErrnoError.With('EACCES', path, 'link');
         }
         return fs.linkSync(path, linkPath);
@@ -608,6 +622,8 @@ export function realpathSync(path, options) {
 }
 realpathSync;
 export function accessSync(path, mode = 0o600) {
+    if (!config.checkAccess)
+        return;
     if (!statSync(path).hasAccess(mode)) {
         throw new ErrnoError(Errno.EACCES);
     }
@@ -621,7 +637,7 @@ export function rmSync(path, options) {
     path = normalizePath(path);
     let stats;
     try {
-        stats = statSync(path);
+        stats = cache.getStats(path) || statSync(path);
     }
     catch (error) {
         if (error.code != 'ENOENT' || !options?.force)
@@ -630,26 +646,31 @@ export function rmSync(path, options) {
     if (!stats) {
         return;
     }
+    cache.setStats(path, stats);
     switch (stats.mode & constants.S_IFMT) {
         case constants.S_IFDIR:
             if (options?.recursive) {
-                for (const entry of readdirSync(path)) {
-                    rmSync(join(path, entry), options);
+                for (const entry of readdirSync(path, { _isIndirect: true })) {
+                    rmSync(join(path, entry), { ...options, _isIndirect: true });
                 }
             }
             rmdirSync(path);
-            return;
+            break;
         case constants.S_IFREG:
         case constants.S_IFLNK:
             unlinkSync(path);
-            return;
+            break;
         case constants.S_IFBLK:
         case constants.S_IFCHR:
         case constants.S_IFIFO:
         case constants.S_IFSOCK:
         default:
+            cache.clearStats();
             throw new ErrnoError(Errno.EPERM, 'File type not supported', path, 'rm');
     }
+    if (!options?._isIndirect) {
+        cache.clearStats();
+    }
 }
 rmSync;
 export function mkdtempSync(prefix, options) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@zenfs/core",
-	"version": "1.1.6",
+	"version": "1.2.0",
 	"description": "A filesystem, anywhere",
 	"funding": {
 		"type": "individual",
@@ -18,6 +18,7 @@
 		"zenfs-test": "scripts/test.js"
 	},
 	"files": [
+		"src",
 		"dist",
 		"tests",
 		"license.md",
@@ -48,7 +49,8 @@
 		"./mixins": "./dist/mixins/index.js",
 		"./path": "./dist/emulation/path.js",
 		"./eslint": "./eslint.shared.js",
-		"./tests/*": "./tests/*"
+		"./tests/*": "./tests/*",
+		"./src/*": "./src/*"
 	},
 	"scripts": {
 		"format": "prettier --write .",

package/scripts/test.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 
 import { execSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { parseArgs } from 'node:util';
 
@@ -8,6 +9,8 @@ const { values: options, positionals } = parseArgs({
 	options: {
 		help: { short: 'h', type: 'boolean', default: false },
 		verbose: { type: 'boolean', default: false },
+		test: { type: 'string' },
+		forceExit: { short: 'f', type: 'boolean', default: false },
 	},
 	allowPositionals: true,
 });
@@ -20,12 +23,22 @@ paths: The setup files to run tests on
 options:
 	--help, -h		Outputs this help message
 	--verbose		Output verbose messages
+	--test			Which test to run
+	--forceExit		Whether to use --test-force-exit
 	`);
 	process.exit();
 }
 
+if (options.verbose) console.debug('Forcing tests to exit (--test-force-exit)');
+
+const testsGlob = join(import.meta.dirname, `../tests/fs/${options.test || '*'}.test.ts`);
+
 for (const setupFile of positionals) {
 	if (options.verbose) console.debug('Running tests for:', setupFile);
 	process.env.SETUP = setupFile;
-	execSync('tsx --test --experimental-test-coverage ' + join(import.meta.dirname, '../tests/fs/*.test.ts'), { stdio: 'inherit' });
+	if (!existsSync(setupFile)) {
+		console.log('ERROR: Skipping non-existent file:', setupFile);
+		continue;
+	}
+	execSync(['tsx --test --experimental-test-coverage', options.forceExit ? '--test-force-exit' : '', testsGlob, process.env.CMD].join(' '), { stdio: 'inherit' });
 }

package/src/backends/backend.ts

@@ -0,0 +1,160 @@
+import type { Entries, RequiredKeys } from 'utilium';
+import { ErrnoError, Errno } from '../error.js';
+import type { FileSystem } from '../filesystem.js';
+import { levenshtein } from '../utils.js';
+
+type OptionType = 'string' | 'number' | 'bigint' | 'boolean' | 'symbol' | 'undefined' | 'object' | 'function';
+
+/**
+ * Resolves the type of Backend.options from the options interface
+ */
+export type OptionsConfig<T> = {
+	[K in keyof T]: {
+		/**
+		 * The basic JavaScript type(s) for this option.
+		 */
+		type: OptionType | readonly OptionType[];
+
+		/**
+		 * Description of the option. Used in error messages and documentation.
+		 */
+		description: string;
+
+		/**
+		 * Whether or not the option is required (optional can be set to null or undefined). Defaults to false.
+		 */
+		required: K extends RequiredKeys<T> ? true : false;
+
+		/**
+		 * A custom validation function to check if the option is valid.
+		 * When async, resolves if valid and rejects if not.
+		 * When sync, it will throw an error if not valid.
+		 */
+		validator?(opt: T[K]): void | Promise<void>;
+	};
+};
+
+/**
+ * Configuration options shared by backends and `Configuration`
+ */
+export interface SharedConfig {
+	/**
+	 * If set, disables the sync cache and sync operations on async file systems.
+	 */
+	disableAsyncCache?: boolean;
+}
+
+/**
+ * A backend
+ */
+export interface Backend<FS extends FileSystem = FileSystem, TOptions extends object = object> {
+	/**
+	 * Create a new instance of the backend
+	 */
+	create(options: TOptions & Partial<SharedConfig>): FS | Promise<FS>;
+
+	/**
+	 * A name to identify the backend.
+	 */
+	name: string;
+
+	/**
+	 * Describes all of the options available for this backend.
+	 */
+	options: OptionsConfig<TOptions>;
+
+	/**
+	 * Whether the backend is available in the current environment.
+	 * It supports checking synchronously and asynchronously
+	 *
+	 * Returns 'true' if this backend is available in the current
+	 * environment. For example, a backend using a browser API will return
+	 * 'false' if the API is unavailable
+	 *
+	 */
+	isAvailable(): boolean | Promise<boolean>;
+}
+
+/**
+ * Gets the options type of a backend
+ * @internal
+ */
+export type OptionsOf<T extends Backend> = T extends Backend<FileSystem, infer TOptions> ? TOptions : never;
+
+/**
+ * Gets the FileSystem type for a backend
+ * @internal
+ */
+export type FilesystemOf<T extends Backend> = T extends Backend<infer FS> ? FS : never;
+
+/** @internal */
+export function isBackend(arg: unknown): arg is Backend {
+	return arg != null && typeof arg == 'object' && 'isAvailable' in arg && typeof arg.isAvailable == 'function' && 'create' in arg && typeof arg.create == 'function';
+}
+
+/**
+ * Checks that `options` object is valid for the file system options.
+ * @internal
+ */
+export async function checkOptions<T extends Backend>(backend: T, options: Record<string, unknown>): Promise<void> {
+	if (typeof options != 'object' || options === null) {
+		throw new ErrnoError(Errno.EINVAL, 'Invalid options');
+	}
+
+	// Check for required options.
+	for (const [optName, opt] of Object.entries(backend.options) as Entries<OptionsConfig<Record<string, any>>>) {
+		const providedValue = options?.[optName];
+
+		if (providedValue === undefined || providedValue === null) {
+			if (!opt.required) {
+				continue;
+			}
+			/* Required option not provided.
+			if any incorrect options provided, which ones are close to the provided one?
+			(edit distance 5 === close)*/
+			const incorrectOptions = Object.keys(options)
+				.filter(o => !(o in backend.options))
+				.map((a: string) => {
+					return { str: a, distance: levenshtein(optName, a) };
+				})
+				.filter(o => o.distance < 5)
+				.sort((a, b) => a.distance - b.distance);
+
+			throw new ErrnoError(
+				Errno.EINVAL,
+				`${backend.name}: Required option '${optName}' not provided.${
+					incorrectOptions.length > 0 ? ` You provided '${incorrectOptions[0].str}', did you mean '${optName}'.` : ''
+				}`
+			);
+		}
+		// Option provided, check type.
+		const typeMatches = Array.isArray(opt.type) ? opt.type.indexOf(typeof providedValue) != -1 : typeof providedValue == opt.type;
+		if (!typeMatches) {
+			throw new ErrnoError(
+				Errno.EINVAL,
+				`${backend.name}: Value provided for option ${optName} is not the proper type. Expected ${
+					Array.isArray(opt.type) ? `one of {${opt.type.join(', ')}}` : (opt.type as string)
+				}, but received ${typeof providedValue}`
+			);
+		}
+
+		if (opt.validator) {
+			await opt.validator(providedValue);
+		}
+		// Otherwise: All good!
+	}
+}
+
+/**
+ * Specifies a file system backend type and its options.
+ *
+ * Individual options can recursively contain BackendConfiguration objects for values that require file systems.
+ *
+ * The configuration for each file system corresponds to that file system's option object passed to its `create()` method.
+ */
+export type BackendConfiguration<T extends Backend> = OptionsOf<T> & Partial<SharedConfig> & { backend: T };
+
+/** @internal */
+export function isBackendConfig<T extends Backend>(arg: unknown): arg is BackendConfiguration<T> {
+	return arg != null && typeof arg == 'object' && 'backend' in arg && isBackend(arg.backend);
+}

package/src/backends/fetch.ts

@@ -0,0 +1,180 @@
+import { Errno, ErrnoError } from '../error.js';
+import type { FileSystemMetadata } from '../filesystem.js';
+import type { Stats } from '../stats.js';
+import type { Backend } from './backend.js';
+import { IndexFS } from './file_index.js';
+import type { IndexData } from './file_index.js';
+
+/**
+ * Asynchronously download a file as a buffer or a JSON object.
+ * Note that the third function signature with a non-specialized type is
+ * invalid, but TypeScript requires it when you specialize string arguments to
+ * constants.
+ * @hidden
+ */
+async function fetchFile(path: string, type: 'buffer'): Promise<Uint8Array>;
+async function fetchFile<T extends object>(path: string, type: 'json'): Promise<T>;
+async function fetchFile<T extends object>(path: string, type: 'buffer' | 'json'): Promise<T | Uint8Array>;
+async function fetchFile<T extends object>(path: string, type: string): Promise<T | Uint8Array> {
+	const response = await fetch(path).catch((e: Error) => {
+		throw new ErrnoError(Errno.EIO, e.message, path);
+	});
+	if (!response.ok) {
+		throw new ErrnoError(Errno.EIO, 'fetch failed: response returned code ' + response.status, path);
+	}
+	switch (type) {
+		case 'buffer': {
+			const arrayBuffer = await response.arrayBuffer().catch((e: Error) => {
+				throw new ErrnoError(Errno.EIO, e.message, path);
+			});
+			return new Uint8Array(arrayBuffer);
+		}
+		case 'json':
+			return response.json().catch((e: Error) => {
+				throw new ErrnoError(Errno.EIO, e.message, path);
+			}) as Promise<T>;
+		default:
+			throw new ErrnoError(Errno.EINVAL, 'Invalid download type: ' + type);
+	}
+}
+
+/**
+ * Configuration options for FetchFS.
+ */
+export interface FetchOptions {
+	/**
+	 * URL to a file index as a JSON file or the file index object itself.
+	 * Defaults to `index.json`.
+	 */
+	index?: string | IndexData;
+
+	/** Used as the URL prefix for fetched files.
+	 * Default: Fetch files relative to the index.
+	 */
+	baseUrl?: string;
+}
+
+/**
+ * A simple filesystem backed by HTTP using the `fetch` API.
+ *
+ *
+ * Index objects look like the following:
+ *
+ * ```json
+ * {
+ * 	"version": 1,
+ * 	"entries": {
+ * 		"/home": { ... },
+ * 		"/home/jvilk": { ... },
+ * 		"/home/james": { ... }
+ * 	}
+ * }
+ * ```
+ *
+ * Each entry contains the stats associated with the file.
+ */
+export class FetchFS extends IndexFS {
+	public readonly baseUrl: string;
+
+	public async ready(): Promise<void> {
+		if (this._isInitialized) {
+			return;
+		}
+		await super.ready();
+
+		if (this._disableSync) {
+			return;
+		}
+
+		/**
+		 * Iterate over all of the files and cache their contents
+		 */
+		for (const [path, stats] of this.index.files()) {
+			await this.getData(path, stats);
+		}
+	}
+
+	public constructor({ index = 'index.json', baseUrl = '' }: FetchOptions) {
+		// prefix url must end in a directory separator.
+		if (baseUrl.at(-1) != '/') {
+			baseUrl += '/';
+		}
+
+		super(typeof index != 'string' ? index : fetchFile<IndexData>(baseUrl + index, 'json'));
+
+		this.baseUrl = baseUrl;
+	}
+
+	public metadata(): FileSystemMetadata {
+		return {
+			...super.metadata(),
+			name: FetchFS.name,
+			readonly: true,
+		};
+	}
+
+	/**
+	 * Preload the `path` into the index.
+	 */
+	public preload(path: string, buffer: Uint8Array): void {
+		const stats = this.index.get(path);
+		if (!stats) {
+			throw ErrnoError.With('ENOENT', path, 'preload');
+		}
+		if (!stats.isFile()) {
+			throw ErrnoError.With('EISDIR', path, 'preload');
+		}
+		stats.size = buffer.length;
+		stats.fileData = buffer;
+	}
+
+	/**
+	 * @todo Be lazier about actually requesting the data?
+	 */
+	protected async getData(path: string, stats: Stats): Promise<Uint8Array> {
+		if (stats.fileData) {
+			return stats.fileData;
+		}
+
+		const data = await fetchFile(this.baseUrl + (path.startsWith('/') ? path.slice(1) : path), 'buffer');
+		stats.fileData = data;
+		return data;
+	}
+
+	protected getDataSync(path: string, stats: Stats): Uint8Array {
+		if (stats.fileData) {
+			return stats.fileData;
+		}
+
+		throw new ErrnoError(Errno.ENODATA, '', path, 'getData');
+	}
+}
+
+const _Fetch = {
+	name: 'Fetch',
+
+	options: {
+		index: {
+			type: ['string', 'object'],
+			required: false,
+			description: 'URL to a file index as a JSON file or the file index object itself, generated with the make-index script. Defaults to `index.json`.',
+		},
+		baseUrl: {
+			type: 'string',
+			required: false,
+			description: 'Used as the URL prefix for fetched files. Default: Fetch files relative to the index.',
+		},
+	},
+
+	isAvailable(): boolean {
+		return typeof globalThis.fetch == 'function';
+	},
+
+	create(options: FetchOptions) {
+		return new FetchFS(options);
+	},
+} as const satisfies Backend<FetchFS, FetchOptions>;
+type _Fetch = typeof _Fetch;
+// eslint-disable-next-line @typescript-eslint/no-empty-object-type
+export interface Fetch extends _Fetch {}
+export const Fetch: Fetch = _Fetch;