@wener/common 2.0.2 → 2.0.5

package/src/fs/server/createNodeFileSystem.ts

@@ -0,0 +1,518 @@
+import {
+	createReadStream as nodeCreateReadStream,
+	createWriteStream as nodeCreateWriteStream,
+	type Stats,
+} from 'node:fs';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import { Readable, type Writable } from 'node:stream';
+import { pathToFileURL } from 'node:url';
+import type {
+	CopyOptions,
+	CreateReadStreamOptions,
+	CreateWriteStreamOptions,
+	IFileStat,
+	IServerFileSystem,
+	MkdirOptions,
+	ReaddirOptions,
+	ReadFileOptions,
+	RenameOptions,
+	RmOptions,
+	StatOptions,
+	WritableData,
+	WriteFileOptions,
+} from '../IFileSystem';
+
+export type INodeFileSystem = IServerFileSystem & {
+	readonly root: string;
+	resolvePath(filePath: string): string;
+};
+
+/**
+ * Creates a Node.js filesystem adapter that implements the IFileSystem interface
+ * @param options Configuration options for the filesystem
+ * @param options.root Optional root directory to restrict all operations within
+ */
+export function createNodeFileSystem(options: { root?: string } = {}): INodeFileSystem {
+	return new NodeFs(options);
+}
+
+type IFS = typeof import('fs/promises');
+
+class NodeFs implements IServerFileSystem, INodeFileSystem {
+	readonly root: string;
+	private readonly fs: IFS;
+
+	constructor({
+		root,
+		fs = fsp,
+	}: {
+		root?: string;
+		fs?: IFS;
+	} = {}) {
+		// Normalize the root path if provided
+		if (root) {
+			this.root = path.resolve(root);
+		} else {
+			this.root = '';
+		}
+		this.fs = fs;
+	}
+
+	// Helper method to handle aborted signals consistently
+	private checkAborted(signal?: AbortSignal): void {
+		if (signal?.aborted) {
+			throw new Error('The operation was aborted');
+		}
+	}
+
+	/**
+	 * Resolves and secures a path by:
+	 * 1. Normalizing the path
+	 * 2. Prepending the root directory if one is set
+	 * 3. Verifying the resulting path is within the root directory
+	 *
+	 * This prevents path traversal attacks.
+	 */
+	resolvePath(filePath: string): string {
+		// Handle empty paths
+		if (!filePath) {
+			return this.root || '';
+		}
+
+		// Normalize to remove any '..' segments and handle slashes
+		const normalizedPath = path.normalize(filePath);
+
+		// If no root is set, just return the normalized path
+		if (!this.root) {
+			return normalizedPath;
+		}
+
+		// Join with root and resolve to absolute path
+		const resolvedPath = path.resolve(path.join(this.root, normalizedPath));
+
+		// Security check: ensure the path is within the root directory
+		if (!resolvedPath.startsWith(this.root)) {
+			throw new Error(`Security violation: Path ${filePath} attempts to access outside of the root directory`);
+		}
+
+		return resolvedPath;
+	}
+
+	/**
+	 * Removes the root prefix from a path for external representation
+	 */
+	private stripRoot(fullPath: string): string {
+		if (!this.root || !fullPath.startsWith(this.root)) {
+			return fullPath;
+		}
+
+		// Remove the root prefix and ensure there's a leading slash
+		let relativePath = fullPath.substring(this.root.length);
+		if (!relativePath) {
+			return '/';
+		}
+
+		if (!relativePath.startsWith('/') && !relativePath.startsWith('\\')) {
+			relativePath = `/${relativePath}`;
+		}
+
+		// Normalize to ensure consistent path separators
+		return path.normalize(relativePath).replace(/\\/g, '/');
+	}
+
+	/**
+	 * Converts a system file stat to our IFileStat interface,
+	 * stripping the root prefix from paths
+	 */
+	private toFileStat(fullPath: string, fsStats: Stats): IFileStat {
+		const normalizedPath = this.stripRoot(fullPath);
+		const directoryPath = path.dirname(normalizedPath);
+		const directory = directoryPath === '.' ? '/' : directoryPath.replace(/\\/g, '/');
+
+		return {
+			directory,
+			path: normalizedPath,
+			name: path.basename(fullPath),
+			kind: fsStats.isDirectory() ? 'directory' : 'file',
+			mtime: fsStats.mtimeMs,
+			size: fsStats.size,
+			meta: {},
+		};
+	}
+
+	async readdir(dir: string, options: ReaddirOptions = {}): Promise<IFileStat[]> {
+		const { fs } = this;
+		const { glob, recursive, depth = 1, kind, hidden = true, signal } = options;
+		this.checkAborted(signal);
+
+		// Resolve the directory path with security checks
+		const resolvedDir = this.resolvePath(dir);
+
+		// Basic file listing
+		const entries = await fs.readdir(resolvedDir, { withFileTypes: true });
+		let results = await Promise.all(
+			entries
+				.filter((entry) => hidden || !entry.name.startsWith('.'))
+				.filter((entry) => !kind || (kind === 'directory' ? entry.isDirectory() : entry.isFile()))
+				.map(async (entry) => {
+					this.checkAborted(signal);
+
+					const entryFullPath = path.join(resolvedDir, entry.name);
+					const stat = await fs.stat(entryFullPath);
+
+					// Convert to external representation
+					return {
+						directory: this.stripRoot(resolvedDir),
+						path: this.stripRoot(entryFullPath),
+						name: entry.name,
+						kind: entry.isDirectory() ? 'directory' : 'file',
+						mtime: stat.mtimeMs,
+						size: stat.size,
+						meta: {},
+					} as IFileStat;
+				}),
+		);
+
+		// Handle recursive option
+		if (recursive || depth > 1) {
+			const subdirs = results.filter((entry) => entry.kind === 'directory');
+
+			for (const subdir of subdirs) {
+				this.checkAborted(signal);
+
+				const maxDepth = recursive ? Infinity : depth - 1;
+				if (maxDepth > 0) {
+					// Need to convert the path back to a full path for the recursive call
+					const _subdirFullPath = this.resolvePath(subdir.path);
+
+					const subEntries = await this.readdir(subdir.path, {
+						...options,
+						depth: maxDepth,
+					});
+					results = [...results, ...subEntries];
+				}
+			}
+		}
+
+		// Handle glob filtering
+		if (glob) {
+			const { matcher } = await import('micromatch');
+			const match = matcher(glob);
+			results = results.filter((entry) => match(entry.path));
+		}
+
+		return results;
+	}
+
+	async stat(filePath: string, options: StatOptions = {}): Promise<IFileStat> {
+		const { fs } = this;
+
+		const { signal } = options;
+		this.checkAborted(signal);
+
+		const resolvedPath = this.resolvePath(filePath);
+
+		try {
+			const stat = await fs.stat(resolvedPath);
+			return this.toFileStat(resolvedPath, stat);
+		} catch (err: any) {
+			if (err.code === 'ENOENT') {
+				throw new Error(`File not found: ${filePath}`);
+			}
+			throw err;
+		}
+	}
+
+	async mkdir(dirPath: string, options: MkdirOptions = {}): Promise<void> {
+		const { fs } = this;
+
+		const { recursive = false, signal } = options;
+		this.checkAborted(signal);
+
+		const resolvedPath = this.resolvePath(dirPath);
+
+		try {
+			await fs.mkdir(resolvedPath, { recursive });
+		} catch (err: any) {
+			if (err.code === 'EEXIST' && recursive) {
+				// Ignore if directory exists and recursive is true
+				return;
+			}
+			throw err;
+		}
+	}
+
+	async readFile(path: string, options?: ReadFileOptions & { encoding: 'text' }): Promise<string>;
+	async readFile(path: string, options?: ReadFileOptions): Promise<Uint8Array>;
+	async readFile(path: string, options: ReadFileOptions = {}): Promise<string | Uint8Array> {
+		const { fs } = this;
+
+		const { encoding = 'binary', signal, onDownloadProgress } = options;
+		this.checkAborted(signal);
+
+		const resolvedPath = this.resolvePath(path);
+
+		try {
+			// Handle progress reporting if needed
+			if (onDownloadProgress) {
+				const stat = await fs.stat(resolvedPath);
+				const stream = this.createReadStream(path, { signal });
+
+				return new Promise((resolve, reject) => {
+					const chunks: Buffer[] = [];
+					let loaded = 0;
+
+					stream.on('data', (chunk) => {
+						chunks.push(Buffer.from(chunk));
+						loaded += chunk.length;
+						onDownloadProgress({ loaded, total: stat.size });
+					});
+
+					stream.on('end', () => {
+						const buffer = Buffer.concat(chunks);
+						if (encoding === 'text') {
+							resolve(buffer.toString('utf-8'));
+						} else {
+							resolve(buffer);
+						}
+					});
+
+					stream.on('error', reject);
+				});
+			}
+
+			// Standard file reading
+			if (encoding === 'text') {
+				return await fs.readFile(resolvedPath, { encoding: 'utf-8' });
+			} else {
+				return await fs.readFile(resolvedPath);
+			}
+		} catch (err: any) {
+			if (err.code === 'ENOENT') {
+				throw new Error(`File not found: ${path}`);
+			}
+			throw err;
+		}
+	}
+
+	async writeFile(path: string, data: WritableData, options: WriteFileOptions = {}): Promise<void> {
+		const { fs } = this;
+
+		const { signal, overwrite = true, onUploadProgress } = options;
+		this.checkAborted(signal);
+
+		const resolvedPath = this.resolvePath(path);
+
+		// Check if file exists and overwrite is false
+		if (!overwrite) {
+			const exists = await this.exists(path);
+			if (exists) {
+				throw new Error(`File already exists: ${path}`);
+			}
+		}
+
+		// Create parent directories if they don't exist
+		const directory = this.getDirectoryName(resolvedPath);
+		if (directory !== resolvedPath) {
+			try {
+				await this.mkdir(this.stripRoot(directory), { recursive: true });
+			} catch (err: any) {
+				// Ignore directory exists error
+				if (err?.code !== 'EEXIST') {
+					throw err;
+				}
+			}
+		}
+
+		if (data instanceof Readable) {
+			let _data = data;
+			return new Promise((resolve, reject) => {
+				const writeStream = this.createWriteStream(path, options);
+				let totalBytes = 0;
+
+				if (onUploadProgress) {
+					_data.on('data', (chunk) => {
+						totalBytes += chunk.length;
+						onUploadProgress({ loaded: totalBytes, total: -1 });
+					});
+				}
+
+				_data.pipe(writeStream);
+
+				writeStream.on('finish', () => resolve());
+				writeStream.on('error', reject);
+
+				if (signal) {
+					signal.addEventListener('abort', () => {
+						_data.destroy();
+						writeStream.destroy();
+						reject(new Error('The operation was aborted'));
+					});
+				}
+			});
+		} else if (data instanceof ReadableStream) {
+			// Handle web ReadableStream
+			const reader = data.getReader();
+			const chunks: Uint8Array[] = [];
+			let loaded = 0;
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+				if (value) {
+					chunks.push(value);
+					loaded += value.length;
+					if (onUploadProgress) {
+						onUploadProgress({ loaded, total: -1 });
+					}
+				}
+			}
+			await fs.writeFile(resolvedPath, Buffer.concat(chunks));
+		} else {
+			// Convert ArrayBuffer/ArrayBufferView to Buffer if necessary
+			let writeData: string | Buffer;
+			if (data instanceof ArrayBuffer) {
+				writeData = Buffer.from(data);
+			} else if (ArrayBuffer.isView(data)) {
+				writeData = Buffer.from(data.buffer, data.byteOffset, data.byteLength);
+			} else {
+				writeData = data;
+			}
+			await fs.writeFile(resolvedPath, writeData);
+		}
+	}
+
+	async rm(path: string, options: RmOptions = {}): Promise<void> {
+		const { fs } = this;
+
+		const { recursive = false, force = false, signal } = options;
+		this.checkAborted(signal);
+
+		const resolvedPath = this.resolvePath(path);
+
+		try {
+			await fs.rm(resolvedPath, { recursive, force });
+		} catch (err: any) {
+			if (force && err.code === 'ENOENT') {
+				return;
+			}
+			throw err;
+		}
+	}
+
+	async rename(oldPath: string, newPath: string, options: RenameOptions = {}): Promise<void> {
+		const { fs } = this;
+
+		const { signal, overwrite = false } = options;
+		this.checkAborted(signal);
+
+		const resolvedOldPath = this.resolvePath(oldPath);
+		const resolvedNewPath = this.resolvePath(newPath);
+
+		// Check if target exists and overwrite is false
+		if (!overwrite) {
+			const exists = await this.exists(newPath);
+			if (exists) {
+				throw new Error(`Destination already exists: ${newPath}`);
+			}
+		}
+
+		try {
+			await fs.rename(resolvedOldPath, resolvedNewPath);
+		} catch (err: any) {
+			if (err.code === 'ENOENT') {
+				throw new Error(`Source file not found: ${oldPath}`);
+			}
+			throw err;
+		}
+	}
+
+	async exists(path: string): Promise<boolean> {
+		const { fs } = this;
+
+		try {
+			await fs.access(this.resolvePath(path));
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
+	async copy(src: string, dest: string, options: CopyOptions = {}): Promise<void> {
+		const { fs } = this;
+
+		const { signal, overwrite = true, shallow = false } = options;
+		this.checkAborted(signal);
+
+		const resolvedSrc = this.resolvePath(src);
+		const resolvedDest = this.resolvePath(dest);
+
+		// Check if source exists
+		try {
+			const stat = await fs.stat(resolvedSrc);
+
+			// Check if destination exists and overwrite is false
+			if (!overwrite) {
+				const exists = await this.exists(dest);
+				if (exists) {
+					throw new Error(`Destination already exists: ${dest}`);
+				}
+			}
+
+			// Create parent directory if it doesn't exist
+			const parentDir = this.getDirectoryName(resolvedDest);
+			await this.mkdir(this.stripRoot(parentDir), { recursive: true });
+
+			// Copy recursively or not based on shallow and if it's a directory
+			await fs.cp(resolvedSrc, resolvedDest, {
+				recursive: !shallow && stat.isDirectory(),
+				force: overwrite,
+			});
+		} catch (err: any) {
+			if (err.code === 'ENOENT') {
+				throw new Error(`Source file not found: ${src}`);
+			}
+			throw err;
+		}
+	}
+
+	createReadStream(filePath: string, options: CreateReadStreamOptions = {}): Readable {
+		const resolvedPath = this.resolvePath(filePath);
+		const { signal, range } = options;
+		const stream = nodeCreateReadStream(resolvedPath, {
+			start: range?.start,
+			end: range?.end,
+		});
+
+		signal?.addEventListener('abort', () => stream.destroy(new Error('The operation was aborted')));
+
+		return stream;
+	}
+
+	createWriteStream(filePath: string, options: CreateWriteStreamOptions = {}): Writable {
+		const resolvedPath = this.resolvePath(filePath);
+		const { signal } = options;
+		const stream = nodeCreateWriteStream(resolvedPath, { flags: options.overwrite === false ? 'wx' : 'w' });
+
+		signal?.addEventListener('abort', () => stream.destroy(new Error('The operation was aborted')));
+
+		return stream;
+	}
+
+	// Helper to extract directory name correctly
+	private getDirectoryName(filePath: string): string {
+		return path.dirname(filePath);
+	}
+
+	getUrl(needle: IFileStat | string) {
+		if (typeof needle === 'object' && needle?.kind !== 'file') {
+			return;
+		}
+		let path = typeof needle === 'string' ? needle : needle.path;
+		if (!path) {
+			return;
+		}
+		// file://
+		return pathToFileURL(this.resolvePath(path)).toString();
+	}
+}

package/src/fs/server/dbfs.test.ts

@@ -0,0 +1,48 @@
+import type { EntityManager } from '@mikro-orm/knex';
+import { beforeEach, describe, test } from 'vitest';
+import { runFileSystemTest } from '../tests/runFileSystemTest';
+import { createDatabaseFileSystem, FileNodeMetaEntity } from './createDatabaseFileSystem';
+import { loadTestDatabase } from './loadTestDatabase';
+
+// Skip: createDatabaseFileSystem imports from @wener/server which is not a dependency of wener-common
+describe.skip('DatabaseFileSystem', () => {
+	let fs: ReturnType<typeof createDatabaseFileSystem>;
+	let em: EntityManager;
+
+	beforeEach(async () => {
+		const { em: entityManager } = await loadTestDatabase();
+		em = entityManager as any; // Type cast to match expected EntityManager type
+
+		fs = createDatabaseFileSystem({
+			getEntityManager: () => em as any,
+		});
+
+		// Setup initial state: ensure root directory exists and create /README.txt
+		const rootNode = await fs.ensureRootNode();
+
+		// Create /README.txt file using EntityManager
+		const content = Buffer.from('Hello');
+		const readmeFile = em.create(FileNodeMetaEntity, {
+			filename: 'README.txt',
+			parent: rootNode,
+			kind: 'file',
+			size: content.length,
+			content: content,
+			atime: new Date(),
+			btime: new Date(),
+			ctime: new Date(),
+			mtime: new Date(),
+		});
+		await em.persistAndFlush(readmeFile);
+	}, 30000); // Increase timeout to 30 seconds
+
+	test('common tests', async () => {
+		await runFileSystemTest(fs, {
+			writableStream: false,
+			readableStream: false,
+			readStream: false,
+			writeStream: false,
+			abort: false,
+		});
+	}, 60000); // Increase timeout to 60 seconds
+});

package/src/fs/server/index.ts

@@ -0,0 +1 @@
+export { createNodeFileSystem } from './createNodeFileSystem';

package/src/fs/server/loadTestDatabase.ts

@@ -0,0 +1,131 @@
+import { inspect } from 'node:util';
+import type { Options as BetterSqliteOptions } from '@mikro-orm/better-sqlite';
+import { Errors, ulid } from '@wener/utils';
+import type { Database } from 'better-sqlite3';
+import { FileNodeContentEntity, FileNodeMetaEntity } from './createDatabaseFileSystem';
+
+export async function loadTestDatabase({ options }: { options?: BetterSqliteOptions } = {}) {
+	const { MikroORM: SqliteMikroORM } = await import('@mikro-orm/better-sqlite');
+
+	const orm = await SqliteMikroORM.init({
+		dbName: ':memory:',
+		entities: [FileNodeContentEntity, FileNodeMetaEntity],
+		discovery: {
+			disableDynamicFileAccess: true,
+			requireEntitiesArray: true,
+		},
+		serialization: {
+			includePrimaryKeys: true,
+			forceObject: true,
+		},
+		findOneOrFailHandler(entityName, where) {
+			throw Errors.NotFound.asError(`未找到数据: ${entityName} ${inspect(where)}`);
+		},
+		findExactlyOneOrFailHandler(entityName, where) {
+			throw Errors.BadRequest.asError(`错误的数据数量: ${entityName} ${inspect(where)}`);
+		},
+		...options,
+	});
+
+	const knex = orm.em.getKnex();
+	{
+		const db: Database = await knex.client.acquireConnection();
+		db.function('ulid', () => ulid());
+		await knex.client.releaseConnection(db);
+	}
+
+	// Execute schema creation
+	for (const stmt of FileSystemSchema.split(';')) {
+		const sql = stmt.trim();
+		if (sql) {
+			await knex.raw(sql);
+		}
+	}
+
+	return {
+		orm,
+		em: orm.em.fork(),
+	};
+}
+
+export const FileSystemSchema = `
+CREATE TABLE IF NOT EXISTS "file_node_meta"
+(
+    -- Base fields
+    "id"         TEXT PRIMARY KEY NOT NULL DEFAULT (ulid()),
+    "tid"        TEXT,
+    "uid"        TEXT,
+    "eid"        TEXT,
+    "created_at" TEXT             NOT NULL DEFAULT (datetime('now')),
+    "updated_at" TEXT             NOT NULL DEFAULT (datetime('now')),
+    "deleted_at" TEXT,
+    "attributes" TEXT,
+    "properties" TEXT,
+    "extensions" TEXT,
+
+    -- File identification
+    "filename"   TEXT             NOT NULL,
+    "size"       INTEGER          NOT NULL DEFAULT 0,
+    "kind"       TEXT             NOT NULL,
+
+    -- Timestamps
+    "atime"      TEXT             NOT NULL,
+    "btime"      TEXT             NOT NULL,
+    "ctime"      TEXT             NOT NULL,
+    "mtime"      TEXT             NOT NULL,
+
+    -- Metadata
+    "metadata"   TEXT             NOT NULL DEFAULT '{}',
+
+    -- Hierarchy
+    "parent_id"  TEXT,
+
+    -- Small file content (for files < 64KB)
+    "content"    BLOB,
+
+    FOREIGN KEY ("parent_id") REFERENCES "file_node_meta" ("id")
+);
+
+CREATE INDEX IF NOT EXISTS "idx_file_node_meta_filename" ON "file_node_meta" ("filename");
+CREATE INDEX IF NOT EXISTS "idx_file_node_meta_parent_id" ON "file_node_meta" ("parent_id");
+CREATE INDEX IF NOT EXISTS "idx_file_node_meta_tid_parent_filename" ON "file_node_meta" ("tid", "parent_id", "filename");
+
+CREATE TABLE IF NOT EXISTS "file_node_content"
+(
+    -- Base fields
+    "id"         TEXT PRIMARY KEY NOT NULL DEFAULT (ulid()),
+    "node_id"    TEXT             NOT NULL UNIQUE,
+    "tid"        TEXT,
+    "uid"        TEXT,
+    "eid"        TEXT,
+    "created_at" TEXT             NOT NULL DEFAULT (datetime('now')),
+    "updated_at" TEXT             NOT NULL DEFAULT (datetime('now')),
+    "deleted_at" TEXT,
+    "attributes" TEXT,
+    "properties" TEXT,
+    "extensions" TEXT,
+
+    -- Content information
+    "size"       INTEGER          NOT NULL,
+    "content"    BLOB,
+
+    -- File properties
+    "mime_type"  TEXT,
+
+    -- Checksums
+    "md5"        TEXT,
+    "sha256"     TEXT,
+
+    -- Content metadata
+    "text"       TEXT,
+    "width"      INTEGER,
+    "height"     INTEGER,
+
+    -- Additional metadata
+    "metadata"   TEXT             NOT NULL DEFAULT '{}',
+
+    FOREIGN KEY ("node_id") REFERENCES "file_node_meta" ("id") ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS "idx_file_node_content_node_id" ON "file_node_content" ("node_id");
+`;