@ryanatkn/gro 0.178.0 → 0.180.0

package/src/lib/filer.ts

@@ -1,5 +1,5 @@
 import {EMPTY_OBJECT} from '@ryanatkn/belt/object.js';
-import {readFileSync, statSync} from 'node:fs';
+import {readFile, stat} from 'node:fs/promises';
 import {dirname, resolve} from 'node:path';
 import type {OmitStrict} from '@ryanatkn/belt/types.js';
 import {isBuiltin} from 'node:module';
@@ -51,6 +51,9 @@ export class Filer {
 	#initing: Promise<void> | undefined;
 	#closing: Promise<void> | undefined;
 
+	#change_queue: Array<WatcherChange> = [];
+	#processing_promise: Promise<void> | null = null;
+
 	constructor(options: FilerOptions = EMPTY_OBJECT) {
 		this.#watch_dir = options.watch_dir ?? watch_dir;
 		this.#watch_dir_options = options.watch_dir_options ?? EMPTY_OBJECT;
@@ -82,9 +85,11 @@ export class Filer {
 			dependencies: new Map(),
 		};
 		this.files.set(id, file);
-		// TODO this may need to be batched/deferred
+		// Defer external file change notification to avoid reentrancy during queue processing
 		if (file.external) {
-			this.#on_change({type: 'add', path: file.id, is_directory: false});
+			queueMicrotask(() => {
+				this.#on_change({type: 'add', path: file.id, is_directory: false});
+			});
 		}
 		return file;
 	};
@@ -117,10 +122,10 @@ export class Filer {
 		this.#initing = this.#init();
 		try {
 			await this.#initing;
-		} catch (err) {
+		} catch (error) {
 			// use shared cleanup logic
 			this.#cleanup();
-			throw err;
+			throw error;
 		} finally {
 			this.#initing = undefined;
 		}
@@ -136,6 +141,9 @@ export class Filer {
 		try {
 			await watcher.init();
 
+			// Wait for any queued changes from init to be processed
+			await this.#drain_queue();
+
 			// check if close() was called during init
 			if (this.#closing) {
 				await watcher.close();
@@ -144,14 +152,14 @@ export class Filer {
 
 			// only set after successful init and not closing
 			this.#watching = watcher;
-		} catch (err) {
+		} catch (error) {
 			// clean up watcher on error, but don't let close error mask init error
 			try {
 				await watcher.close();
 			} catch {
 				// ignore close errors - init error is more important
 			}
-			throw err;
+			throw error;
 		}
 	}
 
@@ -169,6 +177,8 @@ export class Filer {
 		this.#listeners.clear();
 		this.files.clear();
 		this.#watching = undefined;
+		this.#change_queue = [];
+		this.#processing_promise = null;
 		// #initing is handled in finally block of init()
 	}
 
@@ -214,20 +224,22 @@ export class Filer {
 		this.#cleanup();
 	}
 
-	#update(id: PathId): Disknode | null {
+	async #update(id: PathId): Promise<Disknode | null> {
 		const file = this.get_or_create(id);
 
-		let stats: ReturnType<typeof statSync> | null = null;
+		let stats: Awaited<ReturnType<typeof stat>> | null = null;
 		let new_contents: string | null = null; // TODO need to lazily load contents, probably turn `Disknode` into a class
 
 		try {
-			stats = statSync(id);
-			new_contents = readFileSync(id, 'utf8');
-		} catch (err) {
-			if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
-				throw err;
+			[stats, new_contents] = await Promise.all([stat(id), readFile(id, 'utf8')]);
+		} catch (error) {
+			const code = (error as NodeJS.ErrnoException).code;
+			// Treat file as deleted/inaccessible for common error codes
+			if (code === 'ENOENT' || code === 'EACCES' || code === 'EPERM') {
+				// File doesn't exist or is inaccessible, treat as deleted
+			} else {
+				throw error;
 			}
-			// ENOENT: file doesn't exist, treat as deleted
 		}
 
 		const old_mtime = file.mtime;
@@ -245,7 +257,14 @@ export class Filer {
 		const dependencies_before = new Set(file.dependencies.keys());
 		const dependencies_removed = new Set(dependencies_before);
 
-		const imported = file.contents ? parse_imports(file.id, file.contents) : [];
+		let imported: Array<string> = [];
+		if (file.contents) {
+			try {
+				imported = parse_imports(file.id, file.contents);
+			} catch (error) {
+				this.#log?.error('[filer] Failed to parse imports', file.id, error);
+			}
+		}
 		for (const specifier of imported) {
 			if (SVELTEKIT_GLOBAL_SPECIFIER.test(specifier)) continue;
 			const path = map_sveltekit_aliases(specifier, aliases);
@@ -253,7 +272,7 @@ export class Filer {
 			let path_id;
 			// TODO can we replace `resolve_specifier` with `import.meta.resolve` completely now outside of esbuild plugins?
 			if (path[0] === '.' || path[0] === '/') {
-				const resolved = resolve_specifier(path, dir);
+				const resolved = await resolve_specifier(path, dir); // eslint-disable-line no-await-in-loop
 				path_id = resolved.path_id;
 			} else {
 				if (isBuiltin(path)) continue;
@@ -304,8 +323,8 @@ export class Filer {
 		for (const disknode of this.files.values()) {
 			try {
 				listener({type: 'add', path: disknode.id, is_directory: false}, disknode);
-			} catch (err) {
-				this.#log?.error('[filer] Listener error during sync:', err);
+			} catch (error) {
+				this.#log?.error('[filer] Listener error during sync:', error);
 			}
 		}
 	}
@@ -314,8 +333,8 @@ export class Filer {
 		for (const listener of this.#listeners) {
 			try {
 				listener(change, disknode);
-			} catch (err) {
-				this.#log?.error('[filer] Listener error during change notification:', err);
+			} catch (error) {
+				this.#log?.error('[filer] Listener error during change notification:', error);
 			}
 		}
 	}
@@ -335,26 +354,61 @@ export class Filer {
 		// keep watching active even with no listeners, only close() tears down
 	}
 
-	#on_change: WatcherChangeCallback = (change) => {
-		if (this.#closing) return; // ignore changes during close
-		if (change.is_directory) return; // TODO manage directories?
-		let disknode: Disknode | null;
-		switch (change.type) {
-			case 'add':
-			case 'update': {
-				disknode = this.#update(change.path);
-				break;
+	async #drain_queue(): Promise<void> {
+		// Wait for queue to be empty and no active processing
+		while (this.#change_queue.length > 0 || this.#processing_promise) {
+			await this.#process_queue(); // eslint-disable-line no-await-in-loop
+		}
+	}
+
+	async #process_queue(): Promise<void> {
+		// If already processing, return the existing promise
+		if (this.#processing_promise) return this.#processing_promise;
+
+		// Create and track the processing promise
+		this.#processing_promise = this.#do_process_queue();
+
+		try {
+			await this.#processing_promise;
+		} finally {
+			this.#processing_promise = null;
+		}
+	}
+
+	async #do_process_queue(): Promise<void> {
+		while (this.#change_queue.length > 0) {
+			const change = this.#change_queue.shift()!;
+
+			if (this.#closing) continue; // ignore changes during close
+			if (change.is_directory) continue; // TODO manage directories?
+
+			let disknode: Disknode | null;
+			switch (change.type) {
+				case 'add':
+				case 'update': {
+					disknode = await this.#update(change.path); // eslint-disable-line no-await-in-loop
+					break;
+				}
+				case 'delete': {
+					disknode = this.#remove(change.path);
+					break;
+				}
+				default:
+					throw new UnreachableError(change.type);
 			}
-			case 'delete': {
-				disknode = this.#remove(change.path);
-				break;
+
+			if (disknode && this.#listeners.size > 0) {
+				this.#notify_change(change, disknode);
 			}
-			default:
-				throw new UnreachableError(change.type);
-		}
-		if (disknode && this.#listeners.size > 0) {
-			this.#notify_change(change, disknode);
 		}
+	}
+
+	#on_change: WatcherChangeCallback = (change) => {
+		// Enqueue the change (sync callback from chokidar)
+		this.#change_queue.push(change);
+
+		// Start processing if not already running
+		void this.#process_queue();
 	};
 
 	#is_external(id: PathId): boolean {
@@ -372,21 +426,26 @@ export const filter_dependents = (
 	searched: Set<PathId> = new Set(),
 	log?: Logger,
 ): Set<PathId> => {
-	const {dependents} = disknode;
-	for (const dependent_id of dependents.keys()) {
-		if (searched.has(dependent_id)) continue;
-		searched.add(dependent_id);
-		if (!filter || filter(dependent_id)) {
-			results.add(dependent_id);
-		}
-		const dependent_disknode = get_by_id(dependent_id);
-		if (!dependent_disknode) {
-			log?.warn(
-				`[filer.filter_dependents] dependent source file ${dependent_id} not found for ${disknode.id}`,
-			);
-			continue;
+	// Use iterative approach to avoid stack overflow on deep dependency trees
+	const stack = [disknode];
+
+	while (stack.length > 0) {
+		const current = stack.pop()!;
+		for (const dependent_id of current.dependents.keys()) {
+			if (searched.has(dependent_id)) continue;
+			searched.add(dependent_id);
+			if (!filter || filter(dependent_id)) {
+				results.add(dependent_id);
+			}
+			const dependent_disknode = get_by_id(dependent_id);
+			if (!dependent_disknode) {
+				log?.warn(
+					`[filer.filter_dependents] dependent source file ${dependent_id} not found for ${current.id}`,
+				);
+				continue;
+			}
+			stack.push(dependent_disknode);
 		}
-		filter_dependents(dependent_disknode, get_by_id, filter, results, searched);
 	}
 	return results;
 };

package/src/lib/format_file.ts

@@ -19,7 +19,7 @@ export const format_file = async (
 	const final_base_options =
 		base_options !== undefined
 			? base_options
-			: (cached_base_options = load_package_json().prettier as any);
+			: (cached_base_options = (await load_package_json()).prettier as any);
 	let final_options = options;
 	if (options.filepath && !options.parser) {
 		const {filepath, ...rest} = options;

package/src/lib/gen.task.ts

@@ -43,7 +43,7 @@ export const task: Task<Args> = {
 		const input_paths = to_input_paths(raw_input_paths);
 
 		// load all of the gen modules
-		const found = find_genfiles(input_paths, root_dirs, config, timings);
+		const found = await find_genfiles(input_paths, root_dirs, config, timings);
 		if (!found.ok) {
 			if (found.type === 'input_directories_with_no_files') {
 				// TODO maybe let this error like the normal case, but only call `gro gen` if we find gen files? problem is the args would need to be hoisted to callers like `gro sync`

package/src/lib/gen.ts

@@ -4,8 +4,9 @@ import {mkdir, readFile, writeFile} from 'node:fs/promises';
 import type {Result} from '@ryanatkn/belt/result.js';
 import type {Timings} from '@ryanatkn/belt/timings.js';
 import {styleText as st} from 'node:util';
-import {existsSync} from 'node:fs';
 import type {PathId} from '@ryanatkn/belt/path.js';
+import {map_concurrent} from '@ryanatkn/belt/async.js';
+import {fs_search} from '@ryanatkn/belt/fs.js';
 
 import {print_path} from './paths.ts';
 import type {GroConfig} from './gro_config.ts';
@@ -18,7 +19,6 @@ import {
 	type ResolvedInputFile,
 	type ResolvedInputPath,
 } from './input_path.ts';
-import {search_fs} from './search_fs.ts';
 import type {Filer} from './filer.ts';
 import type {InvokeTask} from './task.ts';
 
@@ -210,23 +210,28 @@ export type AnalyzedGenResult =
 			has_changed: true;
 	  };
 
-export const analyze_gen_results = (gen_results: GenResults): Promise<Array<AnalyzedGenResult>> =>
-	Promise.all(
-		gen_results.successes
-			.map((result) => result.files.map((file) => analyze_gen_result(file)))
-			.flat(),
-	);
+export const analyze_gen_results = async (
+	gen_results: GenResults,
+): Promise<Array<AnalyzedGenResult>> => {
+	const files = gen_results.successes.flatMap((result) => result.files);
+	return map_concurrent(files, (file) => analyze_gen_result(file), 10);
+};
 
 export const analyze_gen_result = async (file: GenFile): Promise<AnalyzedGenResult> => {
-	if (!existsSync(file.id)) {
-		return {
-			file,
-			existing_content: null,
-			is_new: true,
-			has_changed: true,
-		};
+	let existing_content: string;
+	try {
+		existing_content = await readFile(file.id, 'utf8');
+	} catch (error) {
+		if (error.code === 'ENOENT') {
+			return {
+				file,
+				existing_content: null,
+				is_new: true,
+				has_changed: true,
+			};
+		}
+		throw error;
 	}
-	const existing_content = await readFile(file.id, 'utf8');
 	return {
 		file,
 		existing_content,
@@ -240,26 +245,25 @@ export const write_gen_results = async (
 	analyzed_gen_results: Array<AnalyzedGenResult>,
 	log: Logger,
 ): Promise<void> => {
-	await Promise.all(
-		gen_results.successes
-			.map((result) =>
-				result.files.map(async (file) => {
-					const analyzed = analyzed_gen_results.find((r) => r.file.id === file.id);
-					if (!analyzed) throw Error('Expected to find analyzed result: ' + file.id);
-					const log_args = [print_path(file.id), 'generated from', print_path(file.origin_id)];
-					if (analyzed.is_new) {
-						log.info('writing new', ...log_args);
-						await mkdir(dirname(file.id), {recursive: true});
-						await writeFile(file.id, file.content);
-					} else if (analyzed.has_changed) {
-						log.info('writing changed', ...log_args);
-						await writeFile(file.id, file.content);
-					} else {
-						log.info('skipping unchanged', ...log_args);
-					}
-				}),
-			)
-			.flat(),
+	const files = gen_results.successes.flatMap((result) => result.files);
+	await map_concurrent(
+		files,
+		async (file) => {
+			const analyzed = analyzed_gen_results.find((r) => r.file.id === file.id);
+			if (!analyzed) throw Error('Expected to find analyzed result: ' + file.id);
+			const log_args = [print_path(file.id), 'generated from', print_path(file.origin_id)];
+			if (analyzed.is_new) {
+				log.info('writing new', ...log_args);
+				await mkdir(dirname(file.id), {recursive: true});
+				await writeFile(file.id, file.content);
+			} else if (analyzed.has_changed) {
+				log.info('writing changed', ...log_args);
+				await writeFile(file.id, file.content);
+			} else {
+				log.info('skipping unchanged', ...log_args);
+			}
+		},
+		10,
 	);
 };
 
@@ -289,17 +293,17 @@ export type FindGenfilesFailure =
 /**
  * Finds modules from input paths. (see `src/lib/input_path.ts` for more)
  */
-export const find_genfiles = (
+export const find_genfiles = async (
 	input_paths: Array<InputPath>,
 	root_dirs: Array<PathId>,
 	config: GroConfig,
 	timings?: Timings,
-): FindGenfilesResult => {
+): Promise<FindGenfilesResult> => {
 	const extensions: Array<string> = [GEN_FILE_PATTERN];
 
 	// Check which extension variation works - if it's a directory, prefer others first!
 	const timing_to_resolve_input_paths = timings?.start('resolve input paths');
-	const {resolved_input_paths, unmapped_input_paths} = resolve_input_paths(
+	const {resolved_input_paths, unmapped_input_paths} = await resolve_input_paths(
 		input_paths,
 		root_dirs,
 		extensions,
@@ -320,15 +324,17 @@ export const find_genfiles = (
 	}
 
 	// Find all of the files for any directories.
-	const timing_to_search_fs = timings?.start('find files');
+	const timing_to_fs_search = timings?.start('find files');
 	const {resolved_input_files, resolved_input_files_by_root_dir, input_directories_with_no_files} =
-		resolve_input_files(resolved_input_paths, (id) =>
-			search_fs(id, {
-				filter: config.search_filters,
-				file_filter: (p) => extensions.some((e) => p.includes(e)),
-			}),
+		await resolve_input_files(
+			resolved_input_paths,
+			async (id) =>
+				await fs_search(id, {
+					filter: config.search_filters,
+					file_filter: (p) => extensions.some((e) => p.includes(e)),
+				}),
 		);
-	timing_to_search_fs?.();
+	timing_to_fs_search?.();
 
 	// Error if any input path has no files. (means we have an empty directory)
 	if (input_directories_with_no_files.length) {

package/src/lib/gro.config.default.ts

@@ -18,7 +18,7 @@ import {load_package_json} from './package_json.ts';
  * - if `src/lib/server/server.ts`, assumes a Node server - needs config
  */
 const config: CreateGroConfig = async (cfg, svelte_config) => {
-	const package_json = load_package_json(); // TODO gets wastefully loaded by some plugins, maybe put in plugin/task context? how does that interact with `map_package_json`?
+	const package_json = await load_package_json(); // TODO gets wastefully loaded by some plugins, maybe put in plugin/task context? how does that interact with `map_package_json`?
 
 	const [has_server_result, has_sveltekit_library_result, has_sveltekit_app_result] =
 		await Promise.all([

package/src/lib/gro_config.ts

@@ -1,5 +1,5 @@
 import {join, resolve} from 'node:path';
-import {existsSync} from 'node:fs';
+import {fs_exists} from '@ryanatkn/belt/fs.js';
 import {identity} from '@ryanatkn/belt/function.js';
 import type {PathFilter, PathId} from '@ryanatkn/belt/path.js';
 import {json_stringify_deterministic} from '@ryanatkn/belt/json.js';
@@ -208,7 +208,7 @@ export const load_gro_config = async (dir = paths.root): Promise<GroConfig> => {
 	);
 
 	const config_path = join(dir, GRO_CONFIG_FILENAME);
-	if (!existsSync(config_path)) {
+	if (!(await fs_exists(config_path))) {
 		// No user config file found, so return the default.
 		return default_config;
 	}

package/src/lib/gro_plugin_gen.ts

@@ -48,7 +48,7 @@ export const gro_plugin_gen = ({
 				// Some parts of the build may have already happened,
 				// making us miss `build` events for gen dependencies,
 				// so we run a full `gen` here even if it's usually wasteful.
-				const found = find_genfiles(input_paths, root_dirs, config);
+				const found = await find_genfiles(input_paths, root_dirs, config);
 				if (found.ok && found.value.resolved_input_files.length > 0) {
 					await gen();
 				}

package/src/lib/gro_plugin_server.ts

@@ -217,8 +217,8 @@ export const gro_plugin_server = ({
 					let build_result;
 					try {
 						build_result = await build_ctx!.rebuild();
-					} catch (err) {
-						log.error('[gro_plugin_server] build failed', err);
+					} catch (error) {
+						log.error('[gro_plugin_server] build failed', error);
 						return;
 					}
 					const {metafile} = build_result;