@zenfs/core 0.9.6 → 0.10.0

package/src/backends/port/fs.ts

@@ -0,0 +1,308 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { ErrnoError, Errno } from '../../error.js';
+import type { Cred } from '../../cred.js';
+import { FileSystem, type FileSystemMetadata, Async } from '../../filesystem.js';
+import { File } from '../../file.js';
+import { Stats, type FileType } from '../../stats.js';
+import { InMemory } from '../InMemory.js';
+import type { SyncStoreFS } from '../SyncStore.js';
+import type { Backend } from '../backend.js';
+import * as RPC from './rpc.js';
+import type { ExtractProperties } from 'utilium';
+import type { FileReadResult } from 'node:fs/promises';
+
+type FileMethods = ExtractProperties<File, (...args: any[]) => Promise<any>>;
+type FileMethod = keyof FileMethods;
+interface FileRequest<TMethod extends FileMethod & string = FileMethod & string> extends RPC.Request<'file', TMethod, Parameters<FileMethods[TMethod]>> {
+	fd: number;
+}
+
+export class PortFile extends File {
+	constructor(
+		public readonly fs: PortFS,
+		public readonly fd: number,
+		public readonly path: string,
+		public position: number
+	) {
+		super();
+	}
+
+	public rpc<const T extends FileMethod & string>(method: T, ...args: Parameters<FileMethods[T]>): Promise<Awaited<ReturnType<FileMethods[T]>>> {
+		return RPC.request<FileRequest<T>, Awaited<ReturnType<FileMethods[T]>>>(
+			{
+				scope: 'file',
+				fd: this.fd,
+				method,
+				args,
+			},
+			this.fs.options
+		);
+	}
+
+	public stat(): Promise<Stats> {
+		return this.rpc('stat');
+	}
+
+	public statSync(): Stats {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.stat');
+	}
+
+	public truncate(len: number): Promise<void> {
+		return this.rpc('truncate', len);
+	}
+
+	public truncateSync(): void {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.truncate');
+	}
+
+	public write(buffer: Uint8Array, offset?: number, length?: number, position?: number): Promise<number> {
+		return this.rpc('write', buffer, offset, length, position);
+	}
+
+	public writeSync(): number {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.write');
+	}
+
+	public async read<TBuffer extends NodeJS.ArrayBufferView>(buffer: TBuffer, offset?: number, length?: number, position?: number): Promise<FileReadResult<TBuffer>> {
+		return (await this.rpc('read', buffer, offset, length, position)) as FileReadResult<TBuffer>;
+	}
+
+	public readSync(): number {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.read');
+	}
+
+	public chown(uid: number, gid: number): Promise<void> {
+		return this.rpc('chown', uid, gid);
+	}
+
+	public chownSync(): void {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.chown');
+	}
+
+	public chmod(mode: number): Promise<void> {
+		return this.rpc('chmod', mode);
+	}
+
+	public chmodSync(): void {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.chmod');
+	}
+
+	public utimes(atime: Date, mtime: Date): Promise<void> {
+		return this.rpc('utimes', atime, mtime);
+	}
+
+	public utimesSync(): void {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.utimes');
+	}
+
+	public _setType(type: FileType): Promise<void> {
+		return this.rpc('_setType', type);
+	}
+
+	public _setTypeSync(): void {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile._setType');
+	}
+
+	public close(): Promise<void> {
+		return this.rpc('close');
+	}
+
+	public closeSync(): void {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.close');
+	}
+
+	public sync(): Promise<void> {
+		return this.rpc('sync');
+	}
+
+	public syncSync(): void {
+		throw ErrnoError.With('ENOSYS', this.path, 'PortFile.sync');
+	}
+}
+
+type FSMethods = ExtractProperties<FileSystem, (...args: any[]) => Promise<any> | FileSystemMetadata>;
+type FSMethod = keyof FSMethods;
+type FSRequest<TMethod extends FSMethod = FSMethod> = RPC.Request<'fs', TMethod, Parameters<FSMethods[TMethod]>>;
+
+/**
+ * PortFS lets you access a ZenFS instance that is running in a port, or the other way around.
+ *
+ * Note that synchronous operations are not permitted on the PortFS, regardless
+ * of the configuration option of the remote FS.
+ */
+export class PortFS extends Async(FileSystem) {
+	public readonly port: RPC.Port;
+
+	/**
+	 * @hidden
+	 */
+	_sync: SyncStoreFS = InMemory.create({ name: 'port-tmpfs' });
+
+	/**
+	 * Constructs a new PortFS instance that connects with ZenFS running on
+	 * the specified port.
+	 */
+	public constructor(public readonly options: RPC.Options) {
+		super();
+		this.port = options.port;
+		RPC.attach<RPC.Response>(this.port, RPC.handleResponse);
+	}
+
+	public metadata(): FileSystemMetadata {
+		return {
+			...super.metadata(),
+			name: 'PortFS',
+		};
+	}
+
+	protected rpc<const T extends FSMethod>(method: T, ...args: Parameters<FSMethods[T]>): Promise<Awaited<ReturnType<FSMethods[T]>>> {
+		return RPC.request<FSRequest<T>, Awaited<ReturnType<FSMethods[T]>>>(
+			{
+				scope: 'fs',
+				method,
+				args,
+			},
+			{ ...this.options, fs: this }
+		);
+	}
+
+	public async ready(): Promise<this> {
+		await this.rpc('ready');
+		await super.ready();
+		return this;
+	}
+
+	public rename(oldPath: string, newPath: string, cred: Cred): Promise<void> {
+		return this.rpc('rename', oldPath, newPath, cred);
+	}
+
+	public async stat(p: string, cred: Cred): Promise<Stats> {
+		return new Stats(await this.rpc('stat', p, cred));
+	}
+
+	public sync(path: string, data: Uint8Array, stats: Readonly<Stats>): Promise<void> {
+		return this.rpc('sync', path, data, stats);
+	}
+	public openFile(p: string, flag: string, cred: Cred): Promise<File> {
+		return this.rpc('openFile', p, flag, cred);
+	}
+	public createFile(p: string, flag: string, mode: number, cred: Cred): Promise<File> {
+		return this.rpc('createFile', p, flag, mode, cred);
+	}
+	public unlink(p: string, cred: Cred): Promise<void> {
+		return this.rpc('unlink', p, cred);
+	}
+	public rmdir(p: string, cred: Cred): Promise<void> {
+		return this.rpc('rmdir', p, cred);
+	}
+	public mkdir(p: string, mode: number, cred: Cred): Promise<void> {
+		return this.rpc('mkdir', p, mode, cred);
+	}
+	public readdir(p: string, cred: Cred): Promise<string[]> {
+		return this.rpc('readdir', p, cred);
+	}
+	public exists(p: string, cred: Cred): Promise<boolean> {
+		return this.rpc('exists', p, cred);
+	}
+	public link(srcpath: string, dstpath: string, cred: Cred): Promise<void> {
+		return this.rpc('link', srcpath, dstpath, cred);
+	}
+}
+
+let nextFd = 0;
+
+const descriptors: Map<number, File> = new Map();
+
+type FileOrFSRequest = FSRequest | FileRequest;
+
+async function handleRequest(port: RPC.Port, fs: FileSystem, request: FileOrFSRequest): Promise<void> {
+	if (!RPC.isMessage(request)) {
+		return;
+	}
+	const { method, args, id, scope, stack } = request;
+
+	let value,
+		error: boolean = false;
+
+	try {
+		switch (scope) {
+			case 'fs':
+				// @ts-expect-error 2556
+				value = await fs[method](...args);
+				if (value instanceof File) {
+					descriptors.set(++nextFd, value);
+					value = {
+						fd: nextFd,
+						path: value.path,
+						position: value.position,
+					};
+				}
+				break;
+			case 'file':
+				const { fd } = request;
+				if (!descriptors.has(fd)) {
+					throw new ErrnoError(Errno.EBADF);
+				}
+				// @ts-expect-error 2556
+				value = await descriptors.get(fd)![method](...args);
+				if (method == 'close') {
+					descriptors.delete(fd);
+				}
+				break;
+			default:
+				return;
+		}
+	} catch (e) {
+		value = e;
+		error = true;
+	}
+
+	port.postMessage({
+		_zenfs: true,
+		scope,
+		id,
+		error,
+		method,
+		stack,
+		value: value instanceof ErrnoError ? value.toJSON() : value,
+	});
+}
+
+export function attachFS(port: RPC.Port, fs: FileSystem): void {
+	RPC.attach<FileOrFSRequest>(port, request => handleRequest(port, fs, request));
+}
+
+export function detachFS(port: RPC.Port, fs: FileSystem): void {
+	RPC.detach<FileOrFSRequest>(port, request => handleRequest(port, fs, request));
+}
+
+export const Port = {
+	name: 'Port',
+
+	options: {
+		port: {
+			type: 'object',
+			required: true,
+			description: 'The target port that you want to connect to',
+			validator(port: RPC.Port) {
+				// Check for a `postMessage` function.
+				if (typeof port?.postMessage != 'function') {
+					throw new ErrnoError(Errno.EINVAL, 'option must be a port.');
+				}
+			},
+		},
+		timeout: {
+			type: 'number',
+			required: false,
+			description: 'How long to wait before the request times out',
+		},
+	},
+
+	async isAvailable(port?: RPC.Port): Promise<boolean> {
+		return true;
+	},
+
+	create(options: RPC.Options) {
+		return new PortFS(options);
+	},
+} satisfies Backend<PortFS, RPC.Options>;

package/src/backends/port/readme.md

@@ -0,0 +1,59 @@
+# Port Backend
+
+A backend for usage with ports and workers. See the examples below.
+
+#### Accessing an FS on a remote Worker from the main thread
+
+Main:
+
+```ts
+import { configure } from '@zenfs/core';
+import { Port } from '@zenfs/port';
+import { Worker } from 'node:worker_threads';
+
+const worker = new Worker('worker.js');
+
+await configure({
+	mounts: {
+		'/worker': {
+			backend: Port,
+			port: worker,
+		},
+	},
+});
+```
+
+Worker:
+
+```ts
+import { InMemory, resolveMountConfig } from '@zenfs/core';
+import { attachFS } from '@zenfs/port';
+import { parentPort } from 'node:worker_threads';
+
+const tmpfs = await resolveMountConfig({ backend: InMemory, name: 'tmp' });
+attachFS(parentPort, tmpfs);
+```
+
+If you are using using web workers, you would use `self` instead of importing `parentPort` in the worker, and would not need to import `Worker` in the main thread.
+
+#### Using with multiple ports on the same thread
+
+```ts
+import { InMemory, fs, resolveMountConfig } from '@zenfs/core';
+import { Port, attachFS } from '@zenfs/port';
+import { MessageChannel } from 'node:worker_threads';
+
+const { port1, port2 } = new MessageChannel();
+
+const tmpfs = await resolveMountConfig({ backend: InMemory, name: 'tmp' });
+attachFS(port2, tmpfs);
+fs.mount('/port', await resolveMountConfig({ backend: Port, port: port1 }));
+console.log('/port');
+
+const content = 'FS is in a port';
+
+await fs.promises.writeFile('/port/test', content);
+
+fs.readFileSync('/tmp/test', 'utf8'); // FS is in a port
+await fs.promises.readFile('/port/test', 'utf8'); // FS is in a port
+```

package/src/backends/port/rpc.ts

@@ -0,0 +1,144 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/// !<reference lib="DOM" />
+import type { TransferListItem } from 'worker_threads';
+import { ErrnoError, Errno, type ErrnoErrorJSON } from '../../error.js';
+import { PortFile, type PortFS } from './fs.js';
+
+type _MessageEvent<T = any> = T | { data: T };
+
+export interface Port {
+	postMessage(value: unknown, transferList?: ReadonlyArray<TransferListItem>): void;
+	on?(event: 'message', listener: (value: unknown) => void): this;
+	off?(event: 'message', listener: (value: unknown) => void): this;
+	addEventListener?(type: 'message', listener: (this: Port, ev: _MessageEvent) => void): void;
+	removeEventListener?(type: 'message', listener: (this: Port, ev: _MessageEvent) => void): void;
+}
+
+export interface Options {
+	/**
+	 * The target port that you want to connect to, or the current port if in a port context.
+	 */
+	port: Port;
+	/**
+	 * How long to wait for a request to complete
+	 */
+	timeout?: number;
+}
+
+/**
+ * An RPC message
+ */
+export interface Message<TScope extends string = string, TMethod extends string = string> {
+	_zenfs: true;
+	scope: TScope;
+	id: string;
+	method: TMethod;
+	stack: string;
+}
+
+export interface Request<TScope extends string = string, TMethod extends string = string, TArgs extends unknown[] = unknown[]> extends Message<TScope, TMethod> {
+	args: TArgs;
+}
+
+export interface Response<TScope extends string = string, TMethod extends string = string, TValue = unknown> extends Message<TScope, TMethod> {
+	error: boolean;
+	value: Awaited<TValue> extends File ? FileData : Awaited<TValue>;
+}
+
+export interface FileData {
+	fd: number;
+	path: string;
+	position: number;
+}
+
+function isFileData(value: unknown): value is FileData {
+	return typeof value == 'object' && value != null && 'fd' in value && 'path' in value && 'position' in value;
+}
+
+export { FileData as File };
+
+// general types
+
+export function isMessage(arg: unknown): arg is Message<string, string> {
+	return typeof arg == 'object' && arg != null && '_zenfs' in arg && !!arg._zenfs;
+}
+
+type _Executor = Parameters<ConstructorParameters<typeof Promise<any>>[0]>;
+
+export interface Executor {
+	resolve: _Executor[0];
+	reject: _Executor[1];
+	fs?: PortFS;
+}
+
+const executors: Map<string, Executor> = new Map();
+
+export function request<const TRequest extends Request, TValue>(
+	request: Omit<TRequest, 'id' | 'stack' | '_zenfs'>,
+	{ port, timeout = 1000, fs }: Partial<Options> & { fs?: PortFS } = {}
+): Promise<TValue> {
+	const stack = new Error().stack!.slice('Error:'.length);
+	if (!port) {
+		throw ErrnoError.With('EINVAL');
+	}
+	return new Promise<TValue>((resolve, reject) => {
+		const id = Math.random().toString(16).slice(10);
+		executors.set(id, { resolve, reject, fs });
+		port.postMessage({ ...request, _zenfs: true, id, stack });
+		setTimeout(() => {
+			const error = new ErrnoError(Errno.EIO, 'RPC Failed');
+			error.stack += stack;
+			reject(error);
+		}, timeout);
+	});
+}
+
+export function handleResponse<const TResponse extends Response>(response: TResponse): void {
+	if (!isMessage(response)) {
+		return;
+	}
+	const { id, value, error, stack } = response;
+	if (!executors.has(id)) {
+		const error = new ErrnoError(Errno.EIO, 'Invalid RPC id:' + id);
+		error.stack += stack;
+		throw error;
+	}
+	const { resolve, reject, fs } = executors.get(id)!;
+	if (error) {
+		const e = ErrnoError.fromJSON(value as ErrnoErrorJSON);
+		e.stack += stack;
+		reject(e);
+		executors.delete(id);
+		return;
+	}
+
+	if (isFileData(value)) {
+		const { fd, path, position } = value;
+		const file = new PortFile(fs!, fd, path, position);
+		resolve(file);
+		executors.delete(id);
+		return;
+	}
+
+	resolve(value);
+	executors.delete(id);
+	return;
+}
+
+export function attach<T extends Message>(port: Port, handler: (message: T) => unknown) {
+	if (!port) {
+		throw ErrnoError.With('EINVAL');
+	}
+	port['on' in port ? 'on' : 'addEventListener']!('message', (message: T | _MessageEvent<T>) => {
+		handler('data' in message ? message.data : message);
+	});
+}
+
+export function detach<T extends Message>(port: Port, handler: (message: T) => unknown) {
+	if (!port) {
+		throw ErrnoError.With('EINVAL');
+	}
+	port['off' in port ? 'off' : 'removeEventListener']!('message', (message: T | _MessageEvent<T>) => {
+		handler('data' in message ? message.data : message);
+	});
+}

package/src/backends/port/store.ts

@@ -0,0 +1,187 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { ErrnoError, Errno } from '../../error.js';
+import { type AsyncStore, type AsyncTransaction, type AsyncStoreOptions, AsyncStoreFS } from '../AsyncStore.js';
+import type { SyncTransaction, SyncStore } from '../SyncStore.js';
+import type { Backend } from '../backend.js';
+import * as RPC from './rpc.js';
+import type { ExtractProperties } from 'utilium';
+
+export class PortStore implements AsyncStore {
+	public readonly port: RPC.Port;
+	public constructor(
+		public readonly options: RPC.Options,
+		public readonly name: string = 'port'
+	) {
+		this.port = options.port;
+		RPC.attach<RPC.Response>(this.port, RPC.handleResponse);
+	}
+
+	public clear(): Promise<void> {
+		return RPC.request(
+			{
+				scope: 'store',
+				method: 'clear',
+				args: [],
+			},
+			this.options
+		);
+	}
+
+	public beginTransaction(): PortTransaction {
+		const id = RPC.request<RPC.Request, number>(
+			{
+				scope: 'store',
+				method: 'beginTransaction',
+				args: [],
+			},
+			this.options
+		);
+		return new PortTransaction(this, id);
+	}
+}
+
+type TxMethods = ExtractProperties<AsyncTransaction, (...args: any[]) => Promise<any>>;
+type TxMethod = keyof TxMethods;
+interface TxRequest<TMethod extends TxMethod = TxMethod> extends RPC.Request<'transaction', TMethod | 'close', Parameters<TxMethods[TMethod]>> {
+	tx: number;
+}
+
+export class PortTransaction implements AsyncTransaction {
+	constructor(
+		public readonly store: PortStore,
+		public readonly id: number | Promise<number>
+	) {}
+
+	public async rpc<const T extends TxMethod>(method: T, ...args: Parameters<TxMethods[T]>): Promise<Awaited<ReturnType<TxMethods[T]>>> {
+		return RPC.request<TxRequest<T>, Awaited<ReturnType<TxMethods[T]>>>(
+			{
+				scope: 'transaction',
+				tx: await this.id,
+				method,
+				args,
+			},
+			this.store.options
+		);
+	}
+
+	public get(key: bigint): Promise<Uint8Array> {
+		return this.rpc('get', key);
+	}
+
+	public async put(key: bigint, data: Uint8Array, overwrite: boolean): Promise<boolean> {
+		return await this.rpc('put', key, data, overwrite);
+	}
+
+	public async remove(key: bigint): Promise<void> {
+		return await this.rpc('remove', key);
+	}
+
+	public async commit(): Promise<void> {
+		return await this.rpc('commit');
+	}
+
+	public async abort(): Promise<void> {
+		return await this.rpc('abort');
+	}
+}
+
+let nextTx = 0;
+
+const transactions: Map<number, AsyncTransaction | SyncTransaction> = new Map();
+
+type StoreOrTxRequest = TxRequest | RPC.Request<'store', keyof ExtractProperties<AsyncStore, (...args: any[]) => any>>;
+
+async function handleRequest(port: RPC.Port, store: AsyncStore | SyncStore, request: StoreOrTxRequest): Promise<void> {
+	if (!RPC.isMessage(request)) {
+		return;
+	}
+	const { method, args, id, scope, stack } = request;
+
+	let value,
+		error: boolean = false;
+
+	try {
+		switch (scope) {
+			case 'store':
+				// @ts-expect-error 2556
+				value = await store[method](...args);
+				if (method == 'beginTransaction') {
+					transactions.set(++nextTx, value!);
+					value = nextTx;
+				}
+				break;
+			case 'transaction':
+				const { tx } = request as TxRequest;
+				if (!transactions.has(tx)) {
+					throw new ErrnoError(Errno.EBADF);
+				}
+				// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+				// @ts-ignore 2556
+				value = await transactions.get(tx)![method](...args);
+				if (method == 'close') {
+					transactions.delete(tx);
+				}
+				break;
+			default:
+				return;
+		}
+	} catch (e) {
+		value = e;
+		error = true;
+	}
+
+	port.postMessage({
+		_zenfs: true,
+		scope,
+		id,
+		error,
+		method,
+		stack,
+		value: value instanceof ErrnoError ? value.toJSON() : value,
+	});
+}
+
+export function attachStore(port: RPC.Port, store: SyncStore | AsyncStore): void {
+	RPC.attach(port, (request: StoreOrTxRequest) => handleRequest(port, store, request));
+}
+
+export function detachStore(port: RPC.Port, store: SyncStore | AsyncStore): void {
+	RPC.detach(port, (request: StoreOrTxRequest) => handleRequest(port, store, request));
+}
+
+export const PortStoreBackend: Backend = {
+	name: 'PortStore',
+
+	options: {
+		port: {
+			type: 'object',
+			description: 'The target port that you want to connect to',
+			validator(port: RPC.Port) {
+				// Check for a `postMessage` function.
+				if (typeof port?.postMessage != 'function') {
+					throw new ErrnoError(Errno.EINVAL, 'option must be a port.');
+				}
+			},
+		},
+	},
+
+	async isAvailable(): Promise<boolean> {
+		if ('WorkerGlobalScope' in globalThis && globalThis instanceof (globalThis as typeof globalThis & { WorkerGlobalScope: any }).WorkerGlobalScope) {
+			// Web Worker
+			return true;
+		}
+
+		try {
+			const worker_threads = await import('node:worker_threads');
+
+			// NodeJS worker
+			return 'Worker' in worker_threads;
+		} catch (e) {
+			return false;
+		}
+	},
+
+	create(options: RPC.Options & AsyncStoreOptions & { name?: string }) {
+		return new AsyncStoreFS({ ...options, store: new PortStore(options, options?.name) });
+	},
+};