evalution 1.0.1

package/dist/cli/index.js

@@ -0,0 +1,871 @@
+import { a as MemoryTraceProvider, i as setupStepCommand, r as CONFIG_FILE_RELATIVE_PATH, t as VercelAISDK } from "../vercel-ai-sdk-CareWPDM.js";
+import module from "node:module";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { serve, upgradeWebSocket } from "@hono/node-server";
+import { serveStatic } from "@hono/node-server/serve-static";
+import { SpanStatusCode, context, trace } from "@opentelemetry/api";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base";
+import { Hono } from "hono";
+import { WebSocketServer } from "ws";
+import { streamSSE } from "hono/streaming";
+import fs$1 from "node:fs";
+import * as pty from "@lydell/node-pty";
+import chokidar from "chokidar";
+import net from "node:net";
+import { spawn } from "node:child_process";
+//#region src/prompt/prompt-registry.ts
+/**
+* Maps globally-usable prompt IDs to the provider-scoped prompt they refer to,
+* so runtime trace spans (which carry only a {@link PromptID}) can be linked
+* back to a concrete prompt.
+*
+* The map is built by scanning every provider's prompts: each prompt registers
+* both its provider-scoped `id` and its author-supplied `globalId` (when set) as
+* keys. A key seen from more than one prompt is **ambiguous** — it is nulled out
+* so it can never resolve to the wrong prompt.
+*/
+var PromptRegistry = class {
+	/** `null` marks an ambiguous key (seen from more than one prompt). */
+	map = /* @__PURE__ */ new Map();
+	/**
+	* Rebuilds the map from scratch by scanning every provider's prompts. Call
+	* after the initial load and whenever a provider's prompts change.
+	*/
+	async rebuild(providers) {
+		const next = /* @__PURE__ */ new Map();
+		for (const [providerId, provider] of providers) {
+			const prompts = await provider.getAllPrompts();
+			for (const prompt of prompts) {
+				this.register(next, prompt.id, {
+					providerId,
+					promptId: prompt.id
+				});
+				if (prompt.globalId) this.register(next, prompt.globalId, {
+					providerId,
+					promptId: prompt.id
+				});
+			}
+		}
+		this.map = next;
+	}
+	register(map, key, value) {
+		if (map.has(key)) map.set(key, null);
+		else map.set(key, value);
+	}
+	/**
+	* Resolves a {@link PromptID} to a provider-scoped reference, or `undefined`
+	* when it can't be resolved.
+	*
+	* The map is always consulted first: it translates a (possibly global) `id`
+	* into the real provider-scoped prompt ID the provider can actually open.
+	* Only when the map has nothing (absent, or ambiguous) do we fall back to
+	* trusting `providerId` together with `id` as already provider-scoped.
+	*/
+	resolve(id, providerId) {
+		const entry = this.map.get(id);
+		if (entry) return entry;
+		return providerId ? {
+			providerId,
+			promptId: id
+		} : void 0;
+	}
+};
+//#endregion
+//#region src/sdk/registry.ts
+/**
+* Every AI SDK offered in manual onboarding, in display order. This is the
+* single source of truth for which SDKs exist and their task ids — adding one
+* to onboarding means giving its adapter a static `setupTask` and listing it
+* here.
+*/
+const AI_SDK_REGISTRY = [VercelAISDK];
+/** Look up a {@link SetupTask} by its id, or `undefined` if none matches. */
+function findSetupTask(taskId) {
+	for (const cls of AI_SDK_REGISTRY) if (cls.setupTask.id === taskId) return cls.setupTask;
+}
+/**
+* Look up a step within a task by both ids, or `undefined` if either is
+* unknown.
+*/
+function findSetupStep(taskId, stepId) {
+	return findSetupTask(taskId)?.steps.find((s) => s.id === stepId);
+}
+//#endregion
+//#region src/server/setup-tasks.ts
+/**
+* Thrown when a requested task or step id does not exist in the registry. The
+* route layer maps this to a 404, distinguishing it from execution failures.
+*/
+var SetupStepNotFoundError = class extends Error {
+	constructor(message) {
+		super(message);
+		this.name = "SetupStepNotFoundError";
+	}
+};
+/**
+* Executes a single onboarding step, resolved from the server-side registry by
+* `taskId`/`stepId`.
+*
+* The client only sends ids; the step definition (file contents, command, ...)
+* comes entirely from {@link AI_SDK_REGISTRY}, so a request can never write
+* arbitrary files or run arbitrary commands.
+*
+* @param rootPath - Absolute path to the project root.
+* @param taskId - Id of the {@link SetupTask} to run a step from.
+* @param stepId - Id of the step within that task.
+* @throws {SetupStepNotFoundError} if the task or step id is unknown.
+* @throws if the step kind is unsupported or execution fails (e.g. the config
+*   file already exists).
+*/
+async function executeSetupStep(rootPath, taskId, stepId) {
+	const step = findSetupStep(taskId, stepId);
+	if (!step) throw new SetupStepNotFoundError(`Unknown step '${stepId}' for task '${taskId}'`);
+	switch (step.kind) {
+		case "create_config": return { path: await writeConfigFile(rootPath, step) };
+		case "run_command":
+		case "install_package": throw new Error(`${step.kind} steps are not yet supported`);
+	}
+}
+/**
+* Returns the onboarding tasks with each step's runtime
+* {@link SetupStepBase.completed | completion status} resolved against the
+* project at `rootPath` (config file present, package installed).
+*
+* @param rootPath - Absolute path to the project root.
+*/
+function resolveSetupTasks(rootPath) {
+	return AI_SDK_REGISTRY.map((cls) => ({
+		...cls.setupTask,
+		steps: cls.setupTask.steps.map((step) => resolveStepStatus(rootPath, step))
+	}));
+}
+/** Adds the runtime `completed` flag to a single step where determinable. */
+function resolveStepStatus(rootPath, step) {
+	switch (step.kind) {
+		case "install_package": return {
+			...step,
+			completed: isPackageInstalled(rootPath, step.package)
+		};
+		case "create_config": return {
+			...step,
+			completed: fs$1.existsSync(path.join(rootPath, step.path))
+		};
+		case "run_command": return step;
+	}
+}
+/**
+* Whether `pkg` is installed for the project at `rootPath`, walking up the
+* directory tree to honour hoisted/workspace `node_modules`.
+*
+* @param rootPath - Absolute path to start the search from.
+* @param pkg - The npm package name to look for.
+*/
+function isPackageInstalled(rootPath, pkg) {
+	let dir = rootPath;
+	while (true) {
+		if (fs$1.existsSync(path.join(dir, "node_modules", pkg, "package.json"))) return true;
+		const parent = path.dirname(dir);
+		if (parent === dir) return false;
+		dir = parent;
+	}
+}
+/**
+* Writes the config file for a `create_config` step, creating parent
+* directories as needed. Refuses to clobber an existing file.
+*/
+async function writeConfigFile(rootPath, step) {
+	const filePath = path.join(rootPath, step.path);
+	try {
+		await fs.access(filePath);
+		throw new Error(`${step.path} already exists`);
+	} catch (err) {
+		if (err?.code !== "ENOENT") throw err;
+	}
+	await fs.mkdir(path.dirname(filePath), { recursive: true });
+	await fs.writeFile(filePath, step.contents, "utf8");
+	return step.path;
+}
+//#endregion
+//#region src/server/api-routes.ts
+function setupRoutes({ app, promptProviders, traceProviders, promptRegistry, hotReloadSubscribers, rootPath, hasConfig, tracer, defaultTraceProviderId }) {
+	const resolveSpanPrompt = (span) => {
+		if (!span.prompt) return span;
+		const resolved = promptRegistry.resolve(span.prompt.id, span.prompt.providerId);
+		if (!resolved) return span;
+		return {
+			...span,
+			prompt: {
+				id: resolved.promptId,
+				providerId: resolved.providerId
+			}
+		};
+	};
+	app.get("/api/config", (c) => c.json({
+		rootPath,
+		configured: hasConfig
+	}));
+	app.get("/api/setup-tasks", (c) => c.json(resolveSetupTasks(rootPath)));
+	app.post("/api/setup-tasks/:taskId/steps/:stepId/execute", async (c) => {
+		const { taskId, stepId } = c.req.param();
+		try {
+			return c.json(await executeSetupStep(rootPath, taskId, stepId));
+		} catch (error) {
+			const status = error instanceof SetupStepNotFoundError ? 404 : 400;
+			return c.json({ error: error.message }, status);
+		}
+	});
+	app.get("/api/providers", (c) => c.json(Array.from(promptProviders.entries()).map(([id, provider]) => ({
+		id,
+		displayName: provider.displayName,
+		description: provider.description,
+		icon: provider.icon,
+		hasAddPrompt: !!provider.addPrompt
+	}))));
+	app.post("/api/providers/:providerId/add-prompt", async (c) => {
+		try {
+			const { providerId } = c.req.param();
+			const provider = promptProviders.get(providerId);
+			if (!provider) return c.json({ error: "Provider not found" }, 404);
+			if (!provider.addPrompt) return c.json({ error: "This provider does not support adding prompts" }, 405);
+			const result = await provider.addPrompt(await c.req.json());
+			if ("fields" in result) return c.json(result);
+			return c.json({
+				...result,
+				providerId
+			});
+		} catch (error) {
+			return c.json({ error: error.message }, 400);
+		}
+	});
+	app.get("/api/providers/:providerId/models", async (c) => {
+		const { providerId } = c.req.param();
+		const provider = promptProviders.get(providerId);
+		if (!provider) return c.json({ error: "Provider not found" }, 404);
+		return c.json(await provider.getModelCatalog?.() ?? { providers: {} });
+	});
+	app.get("/api/providers/:providerId/model-parameters", async (c) => {
+		const { providerId } = c.req.param();
+		const provider = promptProviders.get(providerId);
+		if (!provider) return c.json({ error: "Provider not found" }, 404);
+		return c.json(provider.getModelParameters?.() ?? []);
+	});
+	app.get("/api/prompts", async (c) => {
+		try {
+			const results = await Promise.all(Array.from(promptProviders.entries()).map(async ([providerId, provider]) => {
+				return (await provider.getAllPrompts()).map((prompt) => ({
+					...prompt,
+					providerId
+				}));
+			}));
+			return c.json(results.flat());
+		} catch (error) {
+			return c.json({ error: error.message }, 500);
+		}
+	});
+	app.get("/api/prompts/:providerId/:id", async (c) => {
+		try {
+			const { providerId, id } = c.req.param();
+			const provider = promptProviders.get(providerId);
+			if (!provider) return c.json({ error: "Provider not found" }, 404);
+			const decodedId = Buffer.from(id, "base64url").toString("utf8");
+			const prompt = await provider.getPrompt(decodedId);
+			if (!prompt) return c.json({ error: "Prompt not found" }, 404);
+			return c.json({
+				...prompt,
+				providerId
+			});
+		} catch (error) {
+			return c.json({ error: error.message }, 500);
+		}
+	});
+	app.post("/api/prompts/:providerId/:id/rename", async (c) => {
+		try {
+			const { providerId, id } = c.req.param();
+			const { newName } = await c.req.json();
+			const provider = promptProviders.get(providerId);
+			if (!provider) return c.json({ error: "Provider not found" }, 404);
+			if (!provider.renamePrompt) return c.json({ error: "This provider does not support renaming" }, 405);
+			const decodedId = Buffer.from(id, "base64url").toString("utf8");
+			const updatedPrompt = await provider.renamePrompt(decodedId, newName);
+			return c.json({
+				...updatedPrompt,
+				providerId
+			});
+		} catch (error) {
+			return c.json({ error: error.message }, 400);
+		}
+	});
+	app.post("/api/prompts/:providerId/:id/update", async (c) => {
+		try {
+			const { providerId, id } = c.req.param();
+			const provider = promptProviders.get(providerId);
+			if (!provider) return c.json({ error: "Provider not found" }, 404);
+			if (!provider.updatePromptProperties) return c.json({ error: "This provider does not support editing" }, 405);
+			const decodedId = Buffer.from(id, "base64url").toString("utf8");
+			const updatedPrompt = await provider.updatePromptProperties(decodedId, await c.req.json());
+			return c.json({
+				...updatedPrompt,
+				providerId
+			});
+		} catch (error) {
+			return c.json({ error: error.message }, 400);
+		}
+	});
+	app.post("/api/prompts/:providerId/:id/execute", async (c) => {
+		try {
+			const { providerId, id } = c.req.param();
+			const provider = promptProviders.get(providerId);
+			if (!provider) return c.json({ error: "Provider not found" }, 404);
+			const decodedId = Buffer.from(id, "base64url").toString("utf8");
+			const { functionParams = [] } = await c.req.json().catch(() => ({}));
+			const prompt = await provider.getPrompt(decodedId);
+			if (!prompt) return c.json({ error: "Prompt not found" }, 404);
+			const response = tracer.startActiveSpan(prompt.name, (span) => {
+				const { traceId } = span.spanContext();
+				provider.execute(decodedId, functionParams, false).then(() => {
+					span.setStatus({ code: SpanStatusCode.OK });
+				}, (err) => {
+					console.error("prompt execution failed:", err);
+					span.recordException(err);
+					span.setStatus({
+						code: SpanStatusCode.ERROR,
+						message: err?.error ? JSON.stringify(err.error, null, 2) : err?.message ?? String(err)
+					});
+				}).finally(() => {
+					span.end();
+				});
+				return {
+					traceId,
+					tracerProviderId: defaultTraceProviderId,
+					rootSpanId: span.spanContext().spanId
+				};
+			});
+			return c.json(response);
+		} catch (error) {
+			return c.json({ error: error.message }, 500);
+		}
+	});
+	app.get("/api/trace-providers", (c) => c.json(Array.from(traceProviders.entries()).map(([id, provider]) => ({
+		id,
+		displayName: provider.displayName,
+		description: provider.description
+	}))));
+	app.get("/api/traces", async (c) => {
+		try {
+			const results = await Promise.all(Array.from(traceProviders.values()).map((p) => p.getAllTraces()));
+			return c.json(results.flat());
+		} catch (error) {
+			return c.json({ error: error.message }, 500);
+		}
+	});
+	app.get("/api/traces/:providerId/:id", async (c) => {
+		const { providerId, id } = c.req.param();
+		const provider = traceProviders.get(providerId);
+		if (!provider) return c.json({ error: "Trace provider not found" }, 404);
+		const trace = await provider.getTrace(id);
+		if (!trace) return c.json({ error: "Trace not found" }, 404);
+		return c.json({
+			...trace,
+			spans: trace.spans.map(resolveSpanPrompt)
+		});
+	});
+	app.get("/api/traces/:providerId/:id/events", (c) => {
+		const { providerId, id } = c.req.param();
+		const provider = traceProviders.get(providerId);
+		if (!provider) return c.json({ error: "Trace provider not found" }, 404);
+		const resolveEvent = (event) => "span" in event ? {
+			...event,
+			span: resolveSpanPrompt(event.span)
+		} : event;
+		return streamSSE(c, async (stream) => {
+			await stream.writeSSE({ data: JSON.stringify({ type: "connected" }) });
+			const existing = await provider.getTrace(id);
+			if (existing) for (const span of existing.spans) {
+				const resolved = resolveSpanPrompt(span);
+				const initial = resolved.endTime === void 0 ? {
+					type: "span-start",
+					span: resolved
+				} : {
+					type: "span-end",
+					span: resolved
+				};
+				await stream.writeSSE({ data: JSON.stringify(initial) });
+			}
+			const unsubscribe = provider.subscribeTrace(id, (event) => {
+				stream.writeSSE({ data: JSON.stringify(resolveEvent(event)) });
+			});
+			await new Promise((resolve) => {
+				stream.onAbort(() => {
+					unsubscribe();
+					resolve();
+				});
+			});
+		});
+	});
+	app.get("/api/events", (c) => streamSSE(c, async (stream) => {
+		await stream.writeSSE({ data: JSON.stringify({ type: "connected" }) });
+		const send = (data) => {
+			stream.writeSSE({ data: JSON.stringify(data) });
+		};
+		hotReloadSubscribers.add(send);
+		await new Promise((resolve) => {
+			stream.onAbort(() => {
+				hotReloadSubscribers.delete(send);
+				resolve();
+			});
+		});
+	}));
+}
+//#endregion
+//#region src/server/terminal.ts
+/** Shell used to run a resolved step command, so shell syntax in it works. */
+const SHELL = process.env.SHELL || (process.platform === "win32" ? "powershell.exe" : "bash");
+/**
+* Arguments to run `command` in {@link SHELL}, skipping the user's startup files
+* where the shell allows it. Those rc files (e.g. nvm in `~/.zshrc`) can add
+* seconds of latency before the command even begins, and they are unnecessary
+* here: the PTY inherits the server process's `PATH`, so tools resolve without
+* them.
+*/
+function shellCommandArgs(command) {
+	const shell = SHELL.toLowerCase();
+	if (shell.includes("zsh")) return [
+		"-f",
+		"-c",
+		command
+	];
+	if (shell.includes("bash")) return [
+		"--norc",
+		"--noprofile",
+		"-c",
+		command
+	];
+	return ["-c", command];
+}
+/**
+* Resolves the shell command a terminal should run for a setup step, looking it
+* up in the server's own registry rather than trusting anything from the client.
+* Returns `null` when the step is unknown or is not a runnable command (e.g.
+* `create_config`, which writes a file instead of running in a terminal).
+*/
+function resolveTerminalCommand(taskId, stepId) {
+	const step = findSetupStep(taskId, stepId);
+	if (!step || step.kind === "create_config") return null;
+	return setupStepCommand(step);
+}
+function send(ws, message) {
+	ws.send(JSON.stringify(message));
+}
+/**
+* Registers the interactive-terminal WebSocket route at `/api/terminal`.
+*
+* The client connects with `taskId` and `stepId` query params identifying a
+* setup step; the server resolves the actual command from its own registry
+* (never from the request body) and, once the client signals `start`, spawns it
+* in a PTY rooted at the project. Output is streamed to the client and the
+* client's keystrokes are written to the process, so prompts (e.g. npm's
+* "Ok to proceed?") work. The trust boundary mirrors the step-execute route:
+* the client can only ask to run a step that already exists server-side.
+*/
+function registerTerminalRoute(app, upgradeWebSocket, rootPath) {
+	app.get("/api/terminal", upgradeWebSocket((c) => {
+		const taskId = c.req.query("taskId");
+		const stepId = c.req.query("stepId");
+		let child;
+		const start = (ws, cols, rows) => {
+			if (child) return;
+			const command = taskId && stepId ? resolveTerminalCommand(taskId, stepId) : null;
+			if (!command) {
+				send(ws, {
+					type: "error",
+					message: "Unknown or non-runnable setup step."
+				});
+				ws.close();
+				return;
+			}
+			child = pty.spawn(SHELL, shellCommandArgs(command), {
+				name: "xterm-color",
+				cols: cols || 80,
+				rows: rows || 24,
+				cwd: rootPath,
+				env: process.env
+			});
+			child.onData((data) => send(ws, {
+				type: "data",
+				data
+			}));
+			child.onExit(({ exitCode }) => {
+				send(ws, {
+					type: "exit",
+					code: exitCode
+				});
+				ws.close();
+			});
+		};
+		return {
+			onMessage(evt, ws) {
+				let msg;
+				try {
+					msg = JSON.parse(String(evt.data));
+				} catch {
+					return;
+				}
+				switch (msg.type) {
+					case "start":
+						start(ws, msg.cols, msg.rows);
+						break;
+					case "input":
+						child?.write(msg.data);
+						break;
+					case "resize":
+						child?.resize(msg.cols || 80, msg.rows || 24);
+						break;
+				}
+			},
+			onClose() {
+				child?.kill();
+				child = void 0;
+			}
+		};
+	}));
+}
+//#endregion
+//#region src/server/index.ts
+async function startServer(options) {
+	const { promptProviders, traceProviders, port, rootPath, hasConfig } = options;
+	const promptProviderMap = new Map(promptProviders.map((p) => [p.id, p]));
+	const traceProviderMap = new Map(traceProviders.map((p) => [p.id, p]));
+	const promptRegistry = new PromptRegistry();
+	await promptRegistry.rebuild(promptProviderMap);
+	const tracerProvider = new BasicTracerProvider({ spanProcessors: traceProviders.filter((p) => !!p.getSpanProcessor).map((p) => p.getSpanProcessor()) });
+	trace.setGlobalTracerProvider(tracerProvider);
+	const contextManager = new AsyncLocalStorageContextManager();
+	contextManager.enable();
+	context.setGlobalContextManager(contextManager);
+	const tracer = tracerProvider.getTracer("evalution");
+	const defaultTraceProviderId = traceProviders.find((p) => p instanceof MemoryTraceProvider)?.id ?? traceProviders[0]?.id;
+	if (!defaultTraceProviderId) throw new Error("At least one trace provider must be configured");
+	const app = new Hono();
+	const hotReloadSubscribers = /* @__PURE__ */ new Set();
+	const broadcast = (data) => {
+		for (const send of hotReloadSubscribers) send(data);
+	};
+	setupRoutes({
+		app,
+		promptProviders: promptProviderMap,
+		traceProviders: traceProviderMap,
+		promptRegistry,
+		hotReloadSubscribers,
+		rootPath,
+		hasConfig,
+		tracer,
+		defaultTraceProviderId
+	});
+	registerTerminalRoute(app, upgradeWebSocket, rootPath);
+	const clientRoot = fileURLToPath(new URL("../client/", import.meta.url));
+	app.get("*", serveStatic({ root: clientRoot }));
+	for (const [providerId, provider] of promptProviderMap) if (provider.watch) provider.watch(async (event) => {
+		await promptRegistry.rebuild(promptProviderMap);
+		broadcast({
+			type: "prompt-changed",
+			providerId,
+			event
+		});
+	});
+	for (const [providerId, provider] of traceProviderMap) if (provider.watch) provider.watch((event) => {
+		broadcast({
+			type: "trace-changed",
+			providerId,
+			event
+		});
+	});
+	const wss = new WebSocketServer({ noServer: true });
+	const url = `http://localhost:${port}`;
+	const isDevServer = import.meta.url.includes("/src/");
+	const server = await new Promise((resolve) => {
+		const s = serve({
+			fetch: app.fetch,
+			port,
+			hostname: "0.0.0.0",
+			websocket: { server: wss }
+		}, () => {
+			if (isDevServer) {
+				console.log(`\n✨ Evalution API server running on ${url}`);
+				console.log(`   Frontend dev server: http://localhost:5173\n`);
+			} else console.log(`\n✨ Evalution is running at ${url}\n`);
+			resolve(s);
+		});
+	});
+	const close = () => new Promise((resolve, reject) => {
+		if ("closeAllConnections" in server) server.closeAllConnections();
+		server.close((err) => err ? reject(err) : resolve());
+	});
+	const shutdown = async () => {
+		console.log("\n\nShutting down gracefully...");
+		await close();
+		process.exit(0);
+	};
+	process.on("SIGINT", shutdown);
+	process.on("SIGTERM", shutdown);
+	return {
+		url,
+		close
+	};
+}
+//#endregion
+//#region src/cli/config-loader-hooks.ts
+/**
+* Registers an in-thread module-resolution hook so a project's
+* `.evalution/config.ts` can import the framework by bare specifier
+* (`import { FilePromptProvider } from 'evalution'`) regardless of where it
+* lives or whether evalution is installed in the project's `node_modules`.
+*
+* By default Node resolves `evalution` against the config file's directory,
+* which fails when evalution is run via `npx` (or pointed at another directory)
+* and isn't installed locally. This hook redirects the `evalution` specifier
+* (and its subpaths) to resolve from the running CLI instead, so the config
+* always binds to the same evalution the CLI is executing — no local install
+* required.
+*
+* Uses {@link https://nodejs.org/api/module.html#moduleregisterhooksoptions | `module.registerHooks`}
+* (synchronous, same-thread) rather than `module.register`, so it needs no
+* separate loader file — it survives bundling and behaves identically whether
+* the CLI runs from source (dev) or the compiled bundle (published).
+*
+* @param parentURL - Module URL the `evalution` specifier is resolved against;
+*   pass the CLI's own `import.meta.url`.
+*/
+function registerEvalutionResolver(parentURL) {
+	module.registerHooks({ resolve(specifier, context, nextResolve) {
+		if (specifier === "evalution" || specifier.startsWith("evalution/")) return nextResolve(specifier, {
+			...context,
+			parentURL
+		});
+		return nextResolve(specifier, context);
+	} });
+}
+//#endregion
+//#region src/cli/config-watcher.ts
+/**
+* Watches `rootDir` for `.evalution/config.ts` appearing or changing and runs
+* `onConfig` each time, awaiting it so failures surface rather than vanishing.
+*
+* Used when the server starts without a config file: the onboarding UI (via
+* `POST /api/config/create`) writes the config, and this watcher lets the CLI
+* pick it up and restart with the real config.
+*
+* Rather than watch the not-yet-existing nested config path directly, this
+* watches `rootDir` and ignores everything except the `.evalution` directory
+* and the config file itself, so the watcher reliably fires when the directory
+* and file are created together while never descending into `node_modules` and
+* friends.
+*
+* `onConfig` may be async; it is awaited and any rejection is logged instead of
+* becoming an unhandled rejection. The watcher keeps firing on subsequent
+* changes (overlapping runs are skipped), so if a load fails — e.g. the config
+* has a bad import — the user can fix the file and have it retried. The caller
+* is expected to stop the watcher (via the returned function) once the config
+* has loaded successfully.
+*
+* @param rootDir - The project root that will contain `.evalution/config.ts`.
+* @param onConfig - Run when the config file is created or changed.
+* @returns A function that stops watching.
+*/
+function watchForConfigCreation(rootDir, onConfig) {
+	const configPath = path.join(rootDir, CONFIG_FILE_RELATIVE_PATH);
+	const configDir = path.dirname(configPath);
+	const allowed = new Set([
+		rootDir,
+		configDir,
+		configPath
+	]);
+	const watcher = chokidar.watch(rootDir, {
+		ignoreInitial: true,
+		persistent: true,
+		depth: 2,
+		ignored: (p) => !allowed.has(p)
+	});
+	let running = false;
+	const handle = async (p) => {
+		if (p !== configPath || running) return;
+		running = true;
+		try {
+			await onConfig();
+		} catch (err) {
+			console.error(`Failed to load config from ${configPath}:`, err);
+		} finally {
+			running = false;
+		}
+	};
+	watcher.on("add", handle);
+	watcher.on("change", handle);
+	return () => {
+		watcher.close();
+	};
+}
+//#endregion
+//#region src/cli/find-port.ts
+/** Probes whether `port` can be bound on `host`, resolving to `true` if free. */
+function isPortFree(port, host) {
+	return new Promise((resolve) => {
+		const tester = net.createServer().once("error", () => {
+			resolve(false);
+			tester.close();
+		}).once("listening", () => {
+			tester.close(() => resolve(true));
+		}).listen(port, host);
+	});
+}
+/**
+* Returns the first free port at or after `preferred`, scanning upward. Used by
+* the CLI so `npx evalution` still starts when the default port is already in
+* use instead of crashing with `EADDRINUSE`.
+*
+* @param preferred - The port to try first.
+* @param host - The host to bind against; defaults to `0.0.0.0`.
+* @param maxAttempts - How many sequential ports to try before giving up.
+* @throws If no free port is found within `maxAttempts`.
+*/
+async function findAvailablePort(preferred, host = "0.0.0.0", maxAttempts = 20) {
+	for (let port = preferred; port < preferred + maxAttempts; port++) if (await isPortFree(port, host)) return port;
+	throw new Error(`No free port found in range ${preferred}-${preferred + maxAttempts - 1}`);
+}
+//#endregion
+//#region src/cli/open-browser.ts
+/**
+* Returns the platform-specific command and args used to open `url` in the
+* user's default browser. Exposed separately from {@link openBrowser} so the
+* mapping can be unit-tested without spawning a process.
+*
+* @param url - The URL to open.
+* @param platform - A `process.platform` value; defaults to the current platform.
+*/
+function browserOpenCommand(url, platform = process.platform) {
+	switch (platform) {
+		case "darwin": return {
+			command: "open",
+			args: [url]
+		};
+		case "win32": return {
+			command: "cmd",
+			args: [
+				"/c",
+				"start",
+				"\"\"",
+				url
+			]
+		};
+		default: return {
+			command: "xdg-open",
+			args: [url]
+		};
+	}
+}
+/**
+* Opens `url` in the user's default browser, detached so it never blocks or
+* keeps the CLI alive. Failures (e.g. no browser, headless host) are swallowed
+* — opening the browser is a convenience, not a requirement, and the URL is
+* always printed to the console as a fallback.
+*
+* @param url - The URL to open.
+*/
+function openBrowser(url) {
+	try {
+		const { command, args } = browserOpenCommand(url);
+		const child = spawn(command, args, {
+			stdio: "ignore",
+			detached: true
+		});
+		child.on("error", () => {});
+		child.unref();
+	} catch {}
+}
+//#endregion
+//#region src/cli/index.ts
+registerEvalutionResolver(import.meta.url);
+async function findRootDir(startDir) {
+	let dir = startDir;
+	while (true) {
+		const configPath = path.join(dir, ".evalution", "config.ts");
+		try {
+			await fs.access(configPath);
+			return {
+				rootDir: dir,
+				hasConfig: true
+			};
+		} catch {
+			const parent = path.dirname(dir);
+			if (parent === dir) break;
+			dir = parent;
+		}
+	}
+	return {
+		rootDir: startDir,
+		hasConfig: false
+	};
+}
+async function loadConfig(rootDir) {
+	const configPath = path.join(rootDir, ".evalution", "config.ts");
+	process.chdir(rootDir);
+	const mod = await import(pathToFileURL(configPath).href);
+	console.log(`⚙️ Loaded config from ${configPath}`);
+	return mod.default ?? {};
+}
+function applyDotenv(rootDir) {
+	const envPath = path.join(rootDir, ".env");
+	try {
+		process.loadEnvFile(envPath);
+		console.log(`📄 Loaded environment variables from ${envPath}`);
+	} catch (err) {
+		if (err?.code !== "ENOENT") console.warn(`Warning: failed to load .env from ${envPath}:`, err.message);
+	}
+}
+function startConfiguredServer(rootDir, config, hasConfig, port) {
+	if (config.useDotenv !== false) applyDotenv(rootDir);
+	return startServer({
+		promptProviders: config.promptProviders ?? [],
+		traceProviders: config.traceProviders ?? [new MemoryTraceProvider()],
+		port,
+		rootPath: rootDir,
+		hasConfig
+	});
+}
+async function main() {
+	const args = process.argv.slice(2);
+	if (args.length > 0 && args[0] !== "ui") {
+		console.error(`Unknown command: ${args[0]}`);
+		console.error("Usage: evalution [ui [path]]");
+		process.exit(1);
+	}
+	const pathArg = args[1];
+	const { rootDir, hasConfig } = await findRootDir(pathArg ? path.resolve(pathArg) : process.cwd());
+	let port;
+	if (process.env.PORT) port = parseInt(process.env.PORT, 10);
+	else port = await findAvailablePort(3e3);
+	const maybeOpen = (url) => {
+		if (!process.env.EVALUTION_NO_OPEN) openBrowser(url);
+	};
+	if (hasConfig) {
+		maybeOpen((await startConfiguredServer(rootDir, await loadConfig(rootDir), true, port)).url);
+		return;
+	}
+	let server = await startConfiguredServer(rootDir, {}, false, port);
+	maybeOpen(server.url);
+	console.log(`👀 No config found; watching ${path.join(rootDir, ".evalution", "config.ts")} for creation...`);
+	const stopWatching = watchForConfigCreation(rootDir, async () => {
+		const config = await loadConfig(rootDir);
+		console.log("⚙️ Config loaded; restarting server...");
+		stopWatching();
+		await server.close();
+		server = await startConfiguredServer(rootDir, config, true, port);
+	});
+}
+main().catch((error) => {
+	console.error("Fatal error:", error);
+	process.exit(1);
+});
+//#endregion
+export {};