@fuzdev/gro 0.192.0

package/src/lib/gen_helpers.ts

@@ -0,0 +1,131 @@
+import {resolve} from 'node:path';
+import type {Logger} from '@fuzdev/fuz_util/log.js';
+import type {Timings} from '@fuzdev/fuz_util/timings.js';
+import type {PathId} from '@fuzdev/fuz_util/path.js';
+
+import type {GroConfig} from './gro_config.ts';
+import {filter_dependents, type Filer} from './filer.ts';
+import type {InvokeTask} from './task.ts';
+import {
+	normalize_gen_config,
+	validate_gen_module,
+	type GenContext,
+	type GenDependencies,
+	type GenDependenciesConfig,
+} from './gen.ts';
+import {default_svelte_config} from './svelte_config.ts';
+import {to_root_path} from './paths.ts';
+import {load_module} from './modules.ts';
+
+/**
+ * Check if a file change should trigger a gen file.
+ */
+export const should_trigger_gen = async (
+	gen_file_id: PathId,
+	changed_file_id: PathId,
+	config: GroConfig,
+	filer: Filer,
+	log: Logger,
+	timings: Timings,
+	invoke_task: InvokeTask,
+): Promise<boolean> => {
+	// Always trigger if the gen file itself changed
+	const is_self_change = gen_file_id === changed_file_id;
+	if (is_self_change) return true;
+
+	// Check if gen file depends on the changed file (directly or transitively)
+	const changed_disknode = filer.get_by_id(changed_file_id);
+	let should_bust_cache = false;
+	if (changed_disknode) {
+		const dependents = filter_dependents(
+			changed_disknode,
+			filer.get_by_id,
+			(id) => id === gen_file_id, // filter for just our gen file
+		);
+		should_bust_cache = dependents.has(gen_file_id);
+	}
+
+	// Resolve dependencies (with cache busting if the changed file is a dependency of the gen file)
+	const dependencies = await resolve_gen_dependencies(
+		gen_file_id,
+		changed_file_id,
+		should_bust_cache,
+		config,
+		filer,
+		log,
+		timings,
+		invoke_task,
+	);
+
+	if (!dependencies) return false;
+
+	return (
+		dependencies === 'all' ||
+		dependencies.patterns?.some((p) => p.test(changed_file_id)) ||
+		dependencies.files?.includes(changed_file_id) ||
+		false
+	);
+};
+
+/**
+ * Resolve dependencies for a gen file.
+ * Uses cache-busting only when the gen file itself changes,
+ * otherwise relies on Node's module caching.
+ */
+const resolve_gen_dependencies = async (
+	gen_file_id: string,
+	changed_file_id: PathId | undefined,
+	bust_cache: boolean,
+	config: GroConfig,
+	filer: Filer,
+	log: Logger,
+	timings: Timings,
+	invoke_task: InvokeTask,
+): Promise<GenDependenciesConfig | 'all' | null> => {
+	const result = await load_module(gen_file_id, validate_gen_module, bust_cache);
+
+	if (!result.ok) {
+		if (result.type === 'failed_import') {
+			log.error(`Failed to import ${gen_file_id}:`, result.error);
+		}
+		return null;
+	}
+
+	const gen_config = normalize_gen_config(result.mod.gen);
+	if (!gen_config.dependencies) {
+		return null;
+	}
+
+	let dependencies: GenDependencies | null = gen_config.dependencies;
+	if (typeof dependencies === 'function') {
+		const gen_ctx: GenContext = {
+			config,
+			svelte_config: default_svelte_config,
+			filer,
+			log,
+			timings,
+			invoke_task,
+			origin_id: gen_file_id,
+			origin_path: to_root_path(gen_file_id),
+			changed_file_id,
+		};
+		dependencies = await dependencies(gen_ctx);
+	}
+
+	if (dependencies === null || dependencies === 'all') {
+		return dependencies;
+	}
+
+	// At this point, dependencies is either 'all' or GenDependenciesConfig
+	// For static dependencies, also normalize empty objects
+	if (!dependencies.patterns?.length && !dependencies.files?.length) {
+		return null;
+	}
+
+	// Normalize file paths to absolute paths
+	if (dependencies.files) {
+		dependencies.files = dependencies.files.map((f) => resolve(f));
+	}
+
+	return dependencies;
+};

package/src/lib/github.ts

@@ -0,0 +1,46 @@
+// TODO if this grows at all, use `@octokit/request`,
+// for now it's just calling a single endpoint so we do it manually
+// and we specify just the types we need
+
+import {FetchValueCache, fetch_value} from '@fuzdev/fuz_util/fetch.js';
+import type {Logger} from '@fuzdev/fuz_util/log.js';
+import {z} from 'zod';
+
+export const GITHUB_REPO_MATCHER = /.+github.com\/(.+)\/(.+)/;
+
+export const GithubPullRequest = z.looseObject({
+	url: z.string(),
+	id: z.number(),
+	html_url: z.string(),
+	number: z.number(),
+	user: z.looseObject({
+		login: z.string(),
+	}),
+});
+export type GithubPullRequest = z.infer<typeof GithubPullRequest>;
+
+/**
+ * @see https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-pull-requests-associated-with-a-commit
+ */
+export const github_fetch_commit_prs = async (
+	owner: string,
+	repo: string,
+	commit_sha: string,
+	token?: string,
+	log?: Logger,
+	cache?: FetchValueCache,
+	api_version?: string,
+): Promise<Array<GithubPullRequest> | null> => {
+	const headers = api_version ? new Headers({'x-github-api-version': api_version}) : undefined;
+	const url = `https://api.github.com/repos/${owner}/${repo}/commits/${commit_sha}/pulls`;
+	const fetched = await fetch_value(url, {
+		request: {headers},
+		parse: (v: Array<any>) => v.map((p) => GithubPullRequest.parse(p)),
+		token,
+		cache,
+		return_early_from_cache: true,
+		log,
+	});
+	if (!fetched.ok) return null;
+	return fetched.value;
+};

package/src/lib/gro.config.default.ts

@@ -0,0 +1,42 @@
+import type {CreateGroConfig} from './gro_config.ts';
+import {gro_plugin_sveltekit_library} from './gro_plugin_sveltekit_library.ts';
+import {has_server, gro_plugin_server} from './gro_plugin_server.ts';
+import {gro_plugin_sveltekit_app} from './gro_plugin_sveltekit_app.ts';
+import {has_sveltekit_app, has_sveltekit_library} from './sveltekit_helpers.ts';
+import {gro_plugin_gen} from './gro_plugin_gen.ts';
+import {package_json_load} from './package_json.ts';
+
+// TODO hacky, maybe extract utils?
+
+/**
+ * This is the default config that's passed to `gro.config.ts`
+ * if it exists in the current project, and if not, this is the final config.
+ * It looks at the SvelteKit config and filesystem and tries to do the right thing:
+ *
+ * - if `src/routes`, assumes a SvelteKit frontend - respects `KitConfig.kit.files.routes`
+ * - if `src/lib`, assumes a Node library - respects `KitConfig.kit.files.lib`
+ * - if `src/lib/server/server.ts`, assumes a Node server - needs config
+ */
+const config: CreateGroConfig = async (cfg, svelte_config) => {
+	const package_json = await package_json_load(); // TODO gets wastefully loaded by some plugins, maybe put in plugin/task context? how does that interact with `map_package_json`?
+
+	const [has_server_result, has_sveltekit_library_result, has_sveltekit_app_result] =
+		await Promise.all([
+			has_server(),
+			has_sveltekit_library(package_json, svelte_config),
+			has_sveltekit_app(),
+		]);
+
+	// put things that generate files before SvelteKit so it can see them
+	cfg.plugins = () =>
+		[
+			gro_plugin_gen(),
+			has_server_result.ok ? gro_plugin_server() : null,
+			has_sveltekit_library_result.ok ? gro_plugin_sveltekit_library() : null,
+			has_sveltekit_app_result.ok ? gro_plugin_sveltekit_app() : null,
+		].filter((v) => v !== null);
+
+	return cfg;
+};
+
+export default config;

package/src/lib/gro.ts

@@ -0,0 +1,29 @@
+#!/usr/bin/env -S node --experimental-import-meta-resolve --experimental-strip-types --disable-warning=ExperimentalWarning
+
+// @sync Node options to `$lib/gro_helpers.ts`
+
+import {join} from 'node:path';
+
+import {spawn_result_is_exited} from '@fuzdev/fuz_util/process.js';
+
+import {resolve_gro_module_path, spawn_with_loader} from './gro_helpers.ts';
+
+/*
+
+This file is a loader for the Gro CLI.
+Its only purpose is to import the `invoke.js` script in the correct directory.
+By using `resolve_gro_module_path` it lets the global Gro CLI defer
+to a local installation of Gro if one is available,
+and it also provides special handling for the case
+where we're running Gro inside Gro's own repo for development.
+
+*/
+
+const invoke_path = resolve_gro_module_path('invoke.js');
+
+const loader_path = join(invoke_path, '../loader.js');
+
+const spawned = await spawn_with_loader(loader_path, invoke_path, process.argv.slice(2));
+if (!spawned.ok) {
+	process.exitCode = spawn_result_is_exited(spawned) ? spawned.code : 1;
+}

package/src/lib/gro_config.ts

@@ -0,0 +1,254 @@
+import {join, resolve} from 'node:path';
+import {fs_exists} from '@fuzdev/fuz_util/fs.js';
+import {identity} from '@fuzdev/fuz_util/function.js';
+import type {PathFilter, PathId} from '@fuzdev/fuz_util/path.js';
+import {json_stringify_deterministic} from '@fuzdev/fuz_util/json.js';
+import {hash_secure} from '@fuzdev/fuz_util/hash.js';
+
+import {GRO_DIST_DIR, IS_THIS_GRO, paths} from './paths.ts';
+import {
+	GRO_CONFIG_FILENAME,
+	JS_CLI_DEFAULT,
+	NODE_MODULES_DIRNAME,
+	PM_CLI_DEFAULT,
+	SERVER_DIST_PATH,
+	SVELTEKIT_BUILD_DIRNAME,
+	SVELTEKIT_DIST_DIRNAME,
+} from './constants.ts';
+import create_default_config from './gro.config.default.ts';
+import type {PluginsCreateConfig} from './plugin.ts';
+import type {PackageJsonMapper} from './package_json.ts';
+import type {ParsedSvelteConfig} from './svelte_config.ts';
+import type {FilerOptions} from './filer.ts';
+
+/**
+ * SHA-256 hash of empty string, used for configs without build_cache_config.
+ * This ensures consistent cache behavior when no custom config is provided.
+ */
+export const EMPTY_BUILD_CACHE_CONFIG_HASH =
+	'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855';
+
+/**
+ * The config that users can extend via `gro.config.ts`.
+ * This is exposed to users in places like tasks and genfiles.
+ * @see https://github.com/fuzdev/gro/blob/main/src/docs/config.md
+ */
+export interface GroConfig extends RawGroConfig {
+	/**
+	 * @see https://github.com/fuzdev/gro/blob/main/src/docs/plugin.md
+	 */
+	plugins: PluginsCreateConfig;
+	/**
+	 * Maps the project's `package.json` before writing it to the filesystem.
+	 * The `package_json` argument may be mutated, but the return value is what's used by the caller.
+	 * Returning `null` is a no-op for the caller.
+	 */
+	map_package_json: PackageJsonMapper | null;
+	/**
+	 * The root directories to search for tasks given implicit relative input paths.
+	 * Defaults to `./src/lib`, then the cwd, then the Gro package dist.
+	 */
+	task_root_dirs: Array<PathId>;
+	/**
+	 * When searching the filsystem for tasks and genfiles,
+	 * directories and files are included if they pass all of these filters.
+	 */
+	search_filters: Array<PathFilter>;
+	/**
+	 * The CLI to use that's compatible with `node`.
+	 */
+	js_cli: string;
+	/**
+	 * The CLI to use that's compatible with `npm install` and `npm link`. Defaults to `'npm'`.
+	 */
+	pm_cli: string;
+	/** @default SVELTE_CONFIG_FILENAME */
+	svelte_config_filename?: string;
+	/**
+	 * SHA-256 hash of the user's `build_cache_config` from `gro.config.ts`.
+	 * This is computed during config normalization and the raw value is immediately deleted.
+	 * If no `build_cache_config` was provided, this is the hash of an empty string.
+	 * @see RawGroConfig.build_cache_config
+	 */
+	build_cache_config_hash: string;
+	/**
+	 * Options passed to the `Filer` for file watching and import resolution.
+	 * @see FilerOptions
+	 */
+	filer_options: Partial<FilerOptions> | null;
+}
+
+/**
+ * The relaxed variant of `GroConfig` that users can provide via `gro.config.ts`.
+ * Superset of `GroConfig`.
+ * @see https://github.com/fuzdev/gro/blob/main/src/docs/config.md
+ */
+export interface RawGroConfig {
+	plugins?: PluginsCreateConfig;
+	map_package_json?: PackageJsonMapper | null;
+	task_root_dirs?: Array<string>;
+	search_filters?: PathFilter | Array<PathFilter> | null;
+	js_cli?: string;
+	pm_cli?: string;
+	/**
+	 * Optional object defining custom build inputs for cache invalidation.
+	 * This value is hashed during config normalization and used to detect
+	 * when builds need to be regenerated due to non-source changes.
+	 *
+	 * Use cases:
+	 * - Environment variables baked into build: `{api_url: process.env.PUBLIC_API_URL}`
+	 * - External data files: `{data: fs.readFileSync('data.json', 'utf-8')}`
+	 * - Build feature flags: `{enable_analytics: true}`
+	 *
+	 * Can be a static object or an async function that returns an object.
+	 *
+	 * IMPORTANT: It's safe to include secrets here because they are hashed and `delete`d
+	 * during config normalization. The raw value is never logged or persisted.
+	 */
+	build_cache_config?:
+		| Record<string, unknown>
+		| (() => Record<string, unknown> | Promise<Record<string, unknown>>);
+	/**
+	 * Options passed to the `Filer` for file watching and import resolution.
+	 * @see FilerOptions
+	 */
+	filer_options?: Partial<FilerOptions> | null;
+}
+
+export type CreateGroConfig = (
+	base_config: GroConfig,
+	svelte_config?: ParsedSvelteConfig,
+) => RawGroConfig | Promise<RawGroConfig>;
+
+export const create_empty_gro_config = (): GroConfig => ({
+	plugins: () => [],
+	map_package_json: identity,
+	task_root_dirs: [
+		// TODO maybe disable if no SvelteKit `lib` directory? or other detection to improve defaults
+		paths.lib,
+		IS_THIS_GRO ? null : paths.root,
+		IS_THIS_GRO ? null : GRO_DIST_DIR,
+	].filter((v) => v !== null),
+	search_filters: [(id) => !SEARCH_EXCLUDER_DEFAULT.test(id)],
+	js_cli: JS_CLI_DEFAULT,
+	pm_cli: PM_CLI_DEFAULT,
+	build_cache_config_hash: EMPTY_BUILD_CACHE_CONFIG_HASH,
+	filer_options: null,
+});
+
+/**
+ * The regexp used by default to exclude directories and files
+ * when searching the filesystem for tasks and genfiles.
+ * Customize via `search_filters` in the `GroConfig`.
+ * See the test cases for the exact behavior.
+ */
+export const SEARCH_EXCLUDER_DEFAULT = new RegExp(
+	`(${
+		'(^|/)\\.[^/]+' + // exclude all `.`-prefixed directories
+		// TODO probably change to `pkg.name` instead of this catch-all (also `gro` below)
+		`|(^|/)${NODE_MODULES_DIRNAME}(?!/(@[^/]+/)?gro/${SVELTEKIT_DIST_DIRNAME})` + // exclude `node_modules` unless it's to the Gro directory
+		`|(^|/)${SVELTEKIT_BUILD_DIRNAME}` + // exclude the SvelteKit build directory
+		`|(^|/)(?<!(^|/)gro/)${SVELTEKIT_DIST_DIRNAME}` + // exclude the SvelteKit dist directory unless it's in the Gro directory
+		`|(^|/)${SERVER_DIST_PATH}` + // exclude the Gro server plugin dist directory
+		'|(^|/)test' + // exclude test directories
+		'|(^|/)benchmark' // exclude benchmark directories
+	})($|/)`,
+	'u',
+);
+
+export const EXPORTS_EXCLUDER_DEFAULT = /(\.md|\.(test|ignore)\.|\/(test|ignore)\/)/;
+
+/**
+ * Transforms a `RawGroConfig` to the more strict `GroConfig`.
+ * This allows users to provide a more relaxed config.
+ * Hashes the `build_cache_config` and deletes the raw value for security.
+ */
+export const cook_gro_config = async (raw_config: RawGroConfig): Promise<GroConfig> => {
+	const empty_config = create_empty_gro_config();
+
+	// All of the raw config properties are optional,
+	// so fall back to the empty values when `undefined`.
+	const {
+		plugins = empty_config.plugins,
+		map_package_json = empty_config.map_package_json,
+		task_root_dirs = empty_config.task_root_dirs,
+		search_filters = empty_config.search_filters,
+		js_cli = empty_config.js_cli,
+		pm_cli = empty_config.pm_cli,
+		build_cache_config,
+		filer_options = empty_config.filer_options,
+	} = raw_config;
+
+	// Hash build_cache_config and delete the raw value
+	// IMPORTANT: Raw value may contain secrets - hash it and delete immediately
+	let build_cache_config_hash: string;
+	if (!build_cache_config) {
+		build_cache_config_hash = EMPTY_BUILD_CACHE_CONFIG_HASH;
+	} else {
+		// Resolve if it's a function
+		const resolved =
+			typeof build_cache_config === 'function' ? await build_cache_config() : build_cache_config;
+
+		// Hash the JSON representation with deterministic key ordering
+		build_cache_config_hash = await hash_secure(json_stringify_deterministic(resolved));
+	}
+
+	// Delete the raw value to ensure it doesn't persist in memory
+	delete (raw_config as any).build_cache_config;
+
+	return {
+		plugins,
+		map_package_json,
+		task_root_dirs: task_root_dirs.map((p) => resolve(p)),
+		search_filters: Array.isArray(search_filters)
+			? search_filters
+			: search_filters
+				? [search_filters]
+				: [],
+		js_cli,
+		pm_cli,
+		build_cache_config_hash,
+		filer_options: filer_options ?? null,
+	};
+};
+
+export interface GroConfigModule {
+	readonly default: RawGroConfig | CreateGroConfig;
+}
+
+export const load_gro_config = async (dir = paths.root): Promise<GroConfig> => {
+	const default_config = await cook_gro_config(
+		await create_default_config(create_empty_gro_config()),
+	);
+
+	const config_path = join(dir, GRO_CONFIG_FILENAME);
+	if (!(await fs_exists(config_path))) {
+		// No user config file found, so return the default.
+		return default_config;
+	}
+
+	// Import the user's `gro.config.ts`.
+	const config_module = await import(config_path);
+
+	validate_gro_config_module(config_module, config_path);
+
+	return await cook_gro_config(
+		typeof config_module.default === 'function'
+			? await config_module.default(default_config)
+			: config_module.default,
+	);
+};
+
+export const validate_gro_config_module: (
+	config_module: any,
+	config_path: string,
+) => asserts config_module is GroConfigModule = (config_module, config_path) => {
+	const config = config_module.default;
+	if (!config) {
+		throw Error(`Invalid Gro config module at ${config_path}: expected a default export`);
+	} else if (!(typeof config === 'function' || typeof config === 'object')) {
+		throw Error(
+			`Invalid Gro config module at ${config_path}: the default export must be a function or object`,
+		);
+	}
+};

package/src/lib/gro_helpers.ts

@@ -0,0 +1,108 @@
+import {realpathSync, existsSync} from 'node:fs';
+import {join, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {spawn, type SpawnResult} from '@fuzdev/fuz_util/process.js';
+
+import {JS_CLI_DEFAULT, NODE_MODULES_DIRNAME, SVELTEKIT_DIST_DIRNAME} from './constants.ts';
+
+/*
+
+This module is intended to have minimal dependencies to avoid over-imports in the CLI.
+
+*/
+
+/**
+ * Resolves a path to an internal Gro file.
+ * Prefers any local installation of Gro and falls back to the current CLI context.
+ *
+ * Uses heuristics to find `path`, so may fail in some rare corner cases.
+ * Currently looks for `gro.js` as a sibling to the `path` arg for detection.
+ * If this fails for your usecases, rename `gro.js` or open an issue/PR!
+ *
+ * Used by the CLI and `gro run`.
+ *
+ * case 1:
+ *
+ * We're in a directory that has a local installation of Gro at `node_modules/.bin/gro`.
+ * Use this local version instead of the global.
+ *
+ * case 2:
+ *
+ * We're running Gro inside the Gro repo itself.
+ *
+ * In this case, we use the build directory instead of dist.
+ * There's a paradox here for using Gro inside itself -
+ * ideally we use the dist directory because that's what's shipped,
+ * but the build directory has all of the tests,
+ * and loading two instances of its modules causes problems
+ * like `instanceof` checks failing.
+ * For now we'll just run from build and see if it causes any problems.
+ * There's probably a better design in here somewhere.
+ *
+ * case 3:
+ *
+ * Fall back to invoking Gro from wherever the CLI is being executed.
+ * When using the global CLI, this uses the global Gro installation.
+ *
+ */
+export const resolve_gro_module_path = (path = ''): string => {
+	const gro_bin_path = resolve(NODE_MODULES_DIRNAME, '.bin/gro');
+	// case 1
+	// Prefer any locally installed version of Gro.
+	// This is really confusing if Gro is installed inside Gro itself,
+	// so avoid that when developing Gro.
+	if (existsSync(gro_bin_path)) {
+		return join(realpathSync(gro_bin_path), '..', path);
+	}
+	// case 2
+	// If running Gro inside its own repo, require the local dist.
+	// If the local dist is not yet built it will fall back to the global.
+	if (
+		existsSync(join(SVELTEKIT_DIST_DIRNAME, 'gro.js')) &&
+		existsSync(join(SVELTEKIT_DIST_DIRNAME, path))
+	) {
+		return resolve(SVELTEKIT_DIST_DIRNAME, path);
+	}
+	// case 3
+	// Fall back to the version associated with the running CLI.
+	const file_path = fileURLToPath(import.meta.url);
+	return join(file_path, '..', path);
+};
+
+/**
+ * Runs a file using the Gro loader.
+ *
+ * Uses conditional exports to correctly set up `esm-env` as development by default,
+ * so if you want production set `NODE_ENV=production`.
+ *
+ * @see https://nodejs.org/api/packages.html#conditional-exports
+ *
+ * @param loader_path path to loader
+ * @param invoke_path path to file to spawn with `node`
+ */
+export const spawn_with_loader = (
+	loader_path: string,
+	invoke_path: string,
+	argv: Array<string>,
+	js_cli = JS_CLI_DEFAULT, // TODO source from config when possible
+): Promise<SpawnResult> => {
+	const args = [
+		'--import',
+		// This does the same as `$lib/register.ts` but without the cost of importing another file.
+		`data:text/javascript,
+			import {register} from "node:module";
+			import {pathToFileURL} from "node:url";
+			register("${loader_path}", pathToFileURL("./"));`,
+		// @sync Node options to `$lib/gro.ts`
+		'--experimental-import-meta-resolve', // for `import.meta.resolve`
+		'--experimental-strip-types',
+		'--disable-warning',
+		'ExperimentalWarning',
+	];
+	// In almost all cases we want the exports condition to be `"development"`. Needed for `esm-env`.
+	if (process.env.NODE_ENV !== 'production') {
+		args.push('-C', 'development'); // same as `--conditions`
+	}
+	args.push(invoke_path, ...argv);
+	return spawn(js_cli, args);
+};