@ls-stack/agent-eval 0.28.0 → 0.29.0

package/dist/{cli-BQwRbqsL.mjs → cli-CIc_gBNM.mjs}

@@ -1,25 +1,149 @@
-import { C as normalizeScoreDef, F as getEvalTitle, I as getEvalDisplayStatus, L as deriveScopedSummaryFromCases, N as applyDerivedCallAttributes, S as buildDeclaredColumnDefs, T as createFsCacheStore, V as runSummarySchema, Yn as getEvalRegistry, _ as deriveEvalFreshness, a as getLastRunStatuses, b as resolveEvalDefaultConfig, c as loadPersistedRunSnapshots, d as persistRunState, dt as buildEvalKey, f as recomputeEvalStatusesInRuns, ft as getCaseRowCaseKey, g as resolveArtifactPath, h as resolveTracePresentation, i as generateRunId, l as nextShortIdFromSnapshots, m as runTouchesEval, n as getTargetEvalKeys, o as getLatestRunInfos, ot as resolveApiCallsConfig, p as recomputePersistedCaseStatus, pt as getCaseRowEvalKey, s as loadPersistedRunSnapshot, st as resolveLlmCallsConfig, u as persistCaseDetail, v as loadEvalModule, w as validateCharts, x as loadConfig, y as parseEvalDiscovery } from "./runOrchestration-ClWYWPen.mjs";
-import { createHash } from "node:crypto";
-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
-import { dirname, join, relative, resolve } from "node:path";
+import { B as getEvalDisplayStatus, C as loadConfig, D as createFsCacheStore, E as validateCharts, G as runSummarySchema, L as applyDerivedCallAttributes, S as resolveEvalDefaultConfig, T as normalizeScoreDef, V as deriveScopedSummaryFromCases, _ as buildManualInputDescriptor, _t as getCaseRowEvalKey, a as getLastRunStatuses, b as loadEvalModule, c as loadPersistedRunSnapshots, d as persistRunState, dt as resolveLlmCallsConfig, f as recomputeEvalStatusesInRuns, g as resolveArtifactPath, gt as getCaseRowCaseKey, h as resolveTracePresentation, ht as buildEvalKey, i as generateRunId, l as nextShortIdFromSnapshots, m as runTouchesEval, n as getTargetEvalKeys, o as getLatestRunInfos, p as recomputePersistedCaseStatus, s as loadPersistedRunSnapshot, u as persistCaseDetail, ur as getEvalRegistry, ut as resolveApiCallsConfig, v as parseManualInputValues, w as buildDeclaredColumnDefs, x as parseEvalDiscovery, y as deriveEvalFreshness, z as getEvalTitle } from "./runOrchestration-CIARrLs6.mjs";
+import { createHash, randomUUID } from "node:crypto";
+import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { watch } from "chokidar";
 import { glob } from "glob";
 import { existsSync } from "node:fs";
 import { resultify } from "t-result";
 import { fileURLToPath } from "node:url";
 import { spawn, spawnSync } from "node:child_process";
+//#region ../runner/src/configReload.ts
+/** Coordinates idle-only reloads for `agent-evals.config.ts` in app mode. */
+function createConfigReloadController({ getActiveRunCount, closeRunnerWatchers, loadRunnerState, emitToDiscoveryListeners }) {
+	let watcher;
+	let reloadTimer;
+	let reloadPromise;
+	let state = {
+		status: "idle",
+		activeRunCount: 0,
+		lastChangedAt: null,
+		lastReloadedAt: null
+	};
+	function currentState() {
+		return {
+			...state,
+			activeRunCount: getActiveRunCount()
+		};
+	}
+	function emitReloadEvent() {
+		emitToDiscoveryListeners({
+			type: "config.reload",
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: currentState()
+		});
+	}
+	function setState(patch) {
+		state = {
+			...state,
+			...patch,
+			activeRunCount: getActiveRunCount()
+		};
+		emitReloadEvent();
+	}
+	async function close() {
+		if (reloadTimer !== void 0) {
+			clearTimeout(reloadTimer);
+			reloadTimer = void 0;
+		}
+		const watcherToClose = watcher;
+		watcher = void 0;
+		if (watcherToClose !== void 0) await watcherToClose.close();
+	}
+	async function reloadConfigNow(changedAt) {
+		setState({
+			status: "reloading",
+			lastChangedAt: changedAt
+		});
+		await close();
+		await closeRunnerWatchers();
+		await loadRunnerState();
+		setState({
+			status: "idle",
+			lastChangedAt: changedAt,
+			lastReloadedAt: (/* @__PURE__ */ new Date()).toISOString()
+		});
+	}
+	async function reloadConfig(changedAt) {
+		if (reloadPromise !== void 0) {
+			setState({
+				status: "pending",
+				lastChangedAt: changedAt
+			});
+			await reloadPromise;
+			await reloadIfPendingAndIdle();
+			return;
+		}
+		reloadPromise = reloadConfigNow(changedAt);
+		try {
+			await reloadPromise;
+		} finally {
+			reloadPromise = void 0;
+		}
+	}
+	async function handleConfigChanged() {
+		const changedAt = (/* @__PURE__ */ new Date()).toISOString();
+		if (getActiveRunCount() > 0) {
+			setState({
+				status: "pending",
+				lastChangedAt: changedAt
+			});
+			return;
+		}
+		await reloadConfig(changedAt);
+	}
+	async function reloadIfPendingAndIdle() {
+		if (state.status !== "pending") return;
+		if (getActiveRunCount() > 0) {
+			state = currentState();
+			return;
+		}
+		await reloadConfig(state.lastChangedAt ?? (/* @__PURE__ */ new Date()).toISOString());
+	}
+	async function setupWatcher() {
+		const nextWatcher = watch(resolve(process.cwd(), "agent-evals.config.ts"), {
+			awaitWriteFinish: {
+				stabilityThreshold: 100,
+				pollInterval: 20
+			},
+			ignoreInitial: true,
+			persistent: true
+		});
+		watcher = nextWatcher;
+		const scheduleReload = () => {
+			if (reloadTimer !== void 0) clearTimeout(reloadTimer);
+			reloadTimer = setTimeout(() => {
+				reloadTimer = void 0;
+				handleConfigChanged();
+			}, 50);
+		};
+		nextWatcher.on("change", scheduleReload);
+		nextWatcher.on("add", scheduleReload);
+		nextWatcher.on("unlink", scheduleReload);
+		await new Promise((ready) => {
+			nextWatcher.once("ready", ready);
+		});
+	}
+	return {
+		close,
+		currentState,
+		reloadIfPendingAndIdle,
+		setupWatcher
+	};
+}
+//#endregion
 //#region ../runner/src/evalSummaries.ts
 /** Build the API/UI summary payload for one discovered eval. */
 function buildEvalSummary(params) {
 	const { meta, config, gitState, latestRun, lastRunStatus } = params;
-	const { sourceFingerprint, ...summaryMeta } = meta;
+	const { sourceFingerprint, manualInputDescriptor, requiresManualInput, ...summaryMeta } = meta;
 	const freshness = deriveEvalFreshness({
 		latestRun,
 		gitState,
 		currentEvalSourceFingerprint: sourceFingerprint,
 		staleAfterDays: config.staleAfterDays ?? 14
 	});
-	return {
+	const summary = {
 		...summaryMeta,
 		stale: freshness.stale,
 		outdated: freshness.outdated,
@@ -29,6 +153,8 @@ function buildEvalSummary(params) {
 		currentCommitSha: gitState.commitSha,
 		lastRunStatus
 	};
+	if (manualInputDescriptor && requiresManualInput) summary.manualInput = manualInputDescriptor;
+	return summary;
 }
 /** Write one latest-run snapshot to each targeted eval id. */
 function setLatestRunInfoMap(params) {
@@ -60,6 +186,292 @@ function readGitWorktreeState(workspaceRoot) {
 	return { commitSha: commitResult.status === 0 ? commitResult.stdout : null };
 }
 //#endregion
+//#region ../runner/src/manualInput/discovery.ts
+/**
+* Inspect an eval's `manualInput` config during discovery. Rejects evals that
+* declare both `cases` and `manualInput` and evals whose schema cannot be
+* walked into a wire descriptor.
+*/
+function resolveManualInputDiscovery(params) {
+	const { evalDef, evalId, relativeFilePath } = params;
+	if (!evalDef.manualInput) return { kind: "none" };
+	if (evalDef.cases !== void 0) return {
+		kind: "issue",
+		issue: {
+			type: "manual-input-with-cases",
+			severity: "error",
+			filePath: relativeFilePath,
+			evalId,
+			message: `Eval "${evalId}" in ${relativeFilePath} declares both "cases" and "manualInput". Remove one of them.`
+		}
+	};
+	const descriptorResult = buildManualInputDescriptor(evalDef.manualInput);
+	if (descriptorResult.error) return {
+		kind: "issue",
+		issue: {
+			type: "manual-input-with-cases",
+			severity: "error",
+			filePath: relativeFilePath,
+			evalId,
+			message: `Eval "${evalId}" in ${relativeFilePath} has an unsupported manualInput schema: ${descriptorResult.error.message}`
+		}
+	};
+	return {
+		kind: "ok",
+		requiresManualInput: true,
+		descriptor: descriptorResult.value,
+		config: evalDef.manualInput
+	};
+}
+//#endregion
+//#region ../runner/src/manualInput/files.ts
+const stagedUploadDir = ".agent-evals/manual-input-uploads";
+const mimeTypeByExtension = {
+	".gif": "image/gif",
+	".jpeg": "image/jpeg",
+	".jpg": "image/jpeg",
+	".json": "application/json",
+	".md": "text/markdown",
+	".pdf": "application/pdf",
+	".png": "image/png",
+	".svg": "image/svg+xml",
+	".txt": "text/plain",
+	".webp": "image/webp"
+};
+function toWorkspaceRelativePath(params) {
+	return relative(params.workspaceRoot, params.filePath).replaceAll("\\", "/");
+}
+function isInsideWorkspace(params) {
+	const rel = relative(params.workspaceRoot, params.filePath);
+	return rel === "" || !rel.startsWith("..") && !isAbsolute(rel);
+}
+function sanitizeSegment(value) {
+	const normalized = value.trim().replaceAll(/[^A-Za-z0-9._-]+/g, "-");
+	return normalized.length > 0 ? normalized : "file";
+}
+function sanitizeFileName(value) {
+	const normalized = sanitizeSegment(value);
+	const extension = extname(normalized);
+	if (extension.length === 0) return normalized;
+	return `${normalized.slice(0, -extension.length).replaceAll(".", "-")}${extension}`;
+}
+function inferMimeType(params) {
+	const normalized = params.mimeType?.trim();
+	if (normalized && normalized.length > 0) return normalized;
+	return mimeTypeByExtension[extname(params.name).toLowerCase()] ?? "";
+}
+function hashBytes(bytes) {
+	return createHash("sha256").update(bytes).digest("hex");
+}
+function isRecord$1(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isManualInputFileValue(value) {
+	if (!isRecord$1(value)) return false;
+	return typeof value.name === "string" && typeof value.mimeType === "string" && typeof value.sizeBytes === "number" && typeof value.sha256 === "string" && typeof value.path === "string";
+}
+function isStagedManualInputPath(path) {
+	return path === stagedUploadDir || path.startsWith(`${stagedUploadDir}/`) || path.startsWith(stagedUploadDir + sep);
+}
+/**
+* Persist uploaded manual-input bytes in the workspace staging area and return
+* the JSON-safe metadata used by manual-input schemas.
+*/
+async function stageManualInputFile({ workspaceRoot, bytes, name, mimeType }) {
+	const fileName = sanitizeFileName(name || "uploaded-file");
+	const sha256 = hashBytes(bytes);
+	const dir = resolve(workspaceRoot, stagedUploadDir);
+	await mkdir(dir, { recursive: true });
+	const targetPath = join(dir, `${Date.now().toString(36)}-${randomUUID()}__${sha256.slice(0, 12)}__${fileName}`);
+	await writeFile(targetPath, bytes);
+	return {
+		name: name || fileName,
+		mimeType: inferMimeType({
+			mimeType,
+			name: fileName
+		}),
+		sizeBytes: bytes.byteLength,
+		sha256,
+		path: toWorkspaceRelativePath({
+			workspaceRoot,
+			filePath: targetPath
+		})
+	};
+}
+/**
+* Read a file path supplied by the CLI and stage it as a manual-input file.
+*/
+async function stageManualInputFileFromPath({ workspaceRoot, path, name, mimeType }) {
+	const sourcePath = isAbsolute(path) ? resolve(path) : resolve(workspaceRoot, path);
+	return await stageManualInputFile({
+		workspaceRoot,
+		bytes: new Uint8Array(await readFile(sourcePath)),
+		name: name ?? basename(sourcePath),
+		mimeType: inferMimeType({
+			mimeType,
+			name: name ?? basename(sourcePath)
+		})
+	});
+}
+async function materializeOneManualInputFile(params) {
+	const sourcePath = resolve(params.workspaceRoot, params.value.path);
+	if (!isInsideWorkspace({
+		workspaceRoot: params.workspaceRoot,
+		filePath: sourcePath
+	})) throw new Error(`Manual input file path escapes workspace: ${params.value.path}`);
+	const bytes = new Uint8Array(await readFile(sourcePath));
+	const sha256 = hashBytes(bytes);
+	const fileName = sanitizeFileName(params.value.name || basename(sourcePath));
+	const artifactId = [
+		sanitizeSegment(params.runId),
+		"manual-input",
+		sha256.slice(0, 12),
+		fileName
+	].join("__");
+	const targetPath = join(params.runDir, "artifacts", artifactId);
+	await mkdir(join(params.runDir, "artifacts"), { recursive: true });
+	if (sourcePath !== targetPath) await copyFile(sourcePath, targetPath);
+	if (isStagedManualInputPath(params.value.path)) await resultify(() => rm(sourcePath, { force: true }));
+	return {
+		name: params.value.name,
+		mimeType: inferMimeType({
+			mimeType: params.value.mimeType,
+			name: params.value.name || fileName
+		}),
+		sizeBytes: bytes.byteLength,
+		sha256,
+		path: toWorkspaceRelativePath({
+			workspaceRoot: params.workspaceRoot,
+			filePath: targetPath
+		})
+	};
+}
+async function materializeUnknownValue(params) {
+	if (isManualInputFileValue(params.value)) return await materializeOneManualInputFile({
+		workspaceRoot: params.workspaceRoot,
+		runId: params.runId,
+		runDir: params.runDir,
+		value: params.value
+	});
+	if (Array.isArray(params.value)) return await Promise.all(params.value.map(async (entry) => await materializeUnknownValue({
+		workspaceRoot: params.workspaceRoot,
+		runId: params.runId,
+		runDir: params.runDir,
+		value: entry
+	})));
+	if (isRecord$1(params.value)) {
+		const entries = await Promise.all(Object.entries(params.value).map(async ([key, child]) => {
+			return [key, await materializeUnknownValue({
+				workspaceRoot: params.workspaceRoot,
+				runId: params.runId,
+				runDir: params.runDir,
+				value: child
+			})];
+		}));
+		return Object.fromEntries(entries);
+	}
+	return params.value;
+}
+/**
+* Copy all manual-input file references inside a run request into the run's
+* artifact directory and return a request-safe value with artifact paths.
+*/
+async function materializeManualInputFiles({ workspaceRoot, runId, runDir, value }) {
+	const result = await resultify(() => materializeUnknownValue({
+		workspaceRoot,
+		runId,
+		runDir,
+		value
+	}));
+	if (result.error) return {
+		error: result.error.message,
+		value: null
+	};
+	return {
+		error: null,
+		value: result.value
+	};
+}
+/** Remove stale staged manual-input uploads from previous abandoned runs. */
+async function cleanupStagedManualInputFiles(workspaceRoot) {
+	await resultify(() => rm(resolve(workspaceRoot, stagedUploadDir), {
+		force: true,
+		recursive: true
+	}));
+}
+//#endregion
+//#region ../runner/src/manualInput/validation.ts
+function evalIsTargeted(evalMeta, target) {
+	if (target.evalKeys && target.evalKeys.length > 0) {
+		if (!target.evalKeys.includes(evalMeta.key)) return false;
+	}
+	if (target.evalIds && target.evalIds.length > 0) {
+		if (!target.evalIds.includes(evalMeta.id)) return false;
+	}
+	return true;
+}
+/**
+* Validate the `manualInputs` map carried by a `CreateRunRequest` against the
+* authored Zod schemas of every targeted eval that requires manual input.
+*
+* Pure: takes captured discovery state (eval metas + schema configs) and the
+* request, returns a structured result the server/CLI can format directly.
+*/
+function validateManualInputsForRequest(params) {
+	const { evalMetas, manualInputConfigs, request } = params;
+	const failures = [];
+	const parsed = {};
+	for (const evalMeta of evalMetas) {
+		if (!evalMeta.requiresManualInput) continue;
+		if (!evalIsTargeted(evalMeta, request.target)) continue;
+		const rawValue = request.manualInputs?.[evalMeta.key];
+		if (rawValue === void 0) {
+			failures.push({
+				evalKey: evalMeta.key,
+				evalId: evalMeta.id,
+				reason: "missing",
+				issues: [{
+					path: "",
+					message: `manualInputs is missing an entry for "${evalMeta.key}"`
+				}]
+			});
+			continue;
+		}
+		const config = manualInputConfigs.get(evalMeta.key);
+		if (!config) {
+			failures.push({
+				evalKey: evalMeta.key,
+				evalId: evalMeta.id,
+				reason: "invalid",
+				issues: [{
+					path: "",
+					message: "manualInput schema is unavailable; reload the workspace and try again"
+				}]
+			});
+			continue;
+		}
+		const result = parseManualInputValues(config, rawValue);
+		if (result.error) {
+			failures.push({
+				evalKey: evalMeta.key,
+				evalId: evalMeta.id,
+				reason: "invalid",
+				issues: result.error.issues
+			});
+			continue;
+		}
+		parsed[evalMeta.key] = result.value;
+	}
+	if (failures.length > 0) return {
+		ok: false,
+		failures
+	};
+	return {
+		ok: true,
+		parsed
+	};
+}
+//#endregion
 //#region ../runner/src/recalculateDerivedAttributes.ts
 function getCaseArtifactFileIdForExistingRun(runState, caseRow) {
 	const caseKey = getCaseRowCaseKey(caseRow);
@@ -338,6 +750,9 @@ function getWatchRootsForIncludePatterns(params) {
 }
 //#endregion
 //#region ../runner/src/runner.ts
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
 /** Create an in-memory eval runner bound to the current workspace config. */
 function createRunner({ watchForChanges = true } = {}) {
 	let config;
@@ -347,6 +762,7 @@ function createRunner({ watchForChanges = true } = {}) {
 	let llmCallsConfig = resolveLlmCallsConfig(void 0);
 	let apiCallsConfig = resolveApiCallsConfig(void 0);
 	const evals = /* @__PURE__ */ new Map();
+	const manualInputConfigs = /* @__PURE__ */ new Map();
 	let discoveryIssues = [];
 	const runs = /* @__PURE__ */ new Map();
 	const lastRunStatusMap = /* @__PURE__ */ new Map();
@@ -357,6 +773,12 @@ function createRunner({ watchForChanges = true } = {}) {
 	let runHistoryWatcher;
 	let discoveryRefreshTimer;
 	let runHistoryRefreshTimer;
+	const configReload = createConfigReloadController({
+		getActiveRunCount,
+		closeRunnerWatchers: closeWatchers,
+		loadRunnerState,
+		emitToDiscoveryListeners
+	});
 	function toWorkspaceRelativePath(filePath) {
 		return relative(workspaceRoot, filePath).replaceAll("\\", "/");
 	}
@@ -374,22 +796,7 @@ function createRunner({ watchForChanges = true } = {}) {
 	}
 	const runner = {
 		async init() {
-			config = await loadConfig();
-			workspaceRoot = config.workspaceRoot ?? process.cwd();
-			localStateDir = resolve(workspaceRoot, ".agent-evals");
-			llmCallsConfig = resolveLlmCallsConfig(config.llmCalls);
-			apiCallsConfig = resolveApiCallsConfig(config.apiCalls);
-			await mkdir(localStateDir, { recursive: true });
-			await mkdir(join(localStateDir, "runs"), { recursive: true });
-			cacheStore = createFsCacheStore({
-				workspaceRoot,
-				dir: config.cache?.dir,
-				maxEntriesPerNamespace: config.cache?.maxEntriesPerNamespace ?? config.cache?.maxEntriesPerEval,
-				maxEntriesByNamespace: config.cache?.maxEntriesByNamespace
-			});
-			await loadPersistedRuns();
-			await runner.refreshDiscovery();
-			if (watchForChanges) await setupWatcher();
+			await loadRunnerState();
 		},
 		async listCache() {
 			return cacheStore.list();
@@ -535,6 +942,13 @@ function createRunner({ watchForChanges = true } = {}) {
 			emitDiscoveryEvent();
 			return { deleted: true };
 		},
+		validateManualInputs(request) {
+			return validateManualInputsForRequest({
+				evalMetas: getSortedEvalMetas(),
+				manualInputConfigs,
+				request
+			});
+		},
 		getEvals() {
 			const gitState = readGitWorktreeState(workspaceRoot);
 			const result = [];
@@ -561,6 +975,9 @@ function createRunner({ watchForChanges = true } = {}) {
 		getDiscoveryIssues() {
 			return discoveryIssues;
 		},
+		getConfigReloadState() {
+			return configReload.currentState();
+		},
 		async refreshDiscovery() {
 			const patterns = config.include;
 			const discovered = [];
@@ -572,6 +989,7 @@ function createRunner({ watchForChanges = true } = {}) {
 				discovered.push(...files);
 			}
 			evals.clear();
+			manualInputConfigs.clear();
 			discoveryIssues = [];
 			for (const filePath of discovered) try {
 				const content = await readFile(filePath, "utf-8");
@@ -595,6 +1013,9 @@ function createRunner({ watchForChanges = true } = {}) {
 					let columnDefs = buildDeclaredColumnDefs(void 0, void 0, void 0);
 					let stats;
 					let charts;
+					let manualInputDescriptor;
+					let requiresManualInput = false;
+					const relativeFilePath = toWorkspaceRelativePath(meta.filePath);
 					discoveredEntry?.use((evalDef) => {
 						const defaultConfig = resolveEvalDefaultConfig({
 							evalDef,
@@ -611,8 +1032,25 @@ function createRunner({ watchForChanges = true } = {}) {
 						});
 						for (const warning of validated.warnings) console.warn(warning);
 						charts = validated.charts;
+						const manualInputResult = resolveManualInputDiscovery({
+							evalDef,
+							evalId: meta.id,
+							relativeFilePath
+						});
+						if (manualInputResult.kind === "issue") {
+							discoveryIssues.push(manualInputResult.issue);
+							requiresManualInput = true;
+							return;
+						}
+						if (manualInputResult.kind === "ok") {
+							requiresManualInput = manualInputResult.requiresManualInput;
+							manualInputDescriptor = manualInputResult.descriptor;
+							manualInputConfigs.set(buildEvalKey({
+								filePath: relativeFilePath,
+								evalId: meta.id
+							}), manualInputResult.config);
+						}
 					});
-					const relativeFilePath = toWorkspaceRelativePath(meta.filePath);
 					const key = buildEvalKey({
 						filePath: relativeFilePath,
 						evalId: meta.id
@@ -627,7 +1065,9 @@ function createRunner({ watchForChanges = true } = {}) {
 						columnDefs,
 						caseCount: null,
 						stats,
-						charts
+						charts,
+						manualInputDescriptor,
+						requiresManualInput
 					});
 				}
 			} catch {}
@@ -673,11 +1113,27 @@ function createRunner({ watchForChanges = true } = {}) {
 				childProcess: void 0,
 				childTerminalReceived: false
 			};
+			await mkdir(runDir, { recursive: true });
+			await mkdir(join(runDir, "traces"), { recursive: true });
+			await mkdir(join(runDir, "artifacts"), { recursive: true });
+			await mkdir(join(runDir, "case-details"), { recursive: true });
+			const materializedRequest = { ...request };
+			if (request.manualInputs !== void 0) {
+				const materialized = await materializeManualInputFiles({
+					workspaceRoot,
+					runId,
+					runDir,
+					value: request.manualInputs
+				});
+				if (materialized.error !== null) throw new Error(materialized.error);
+				if (!isRecord(materialized.value)) throw new Error("Materialized manual inputs must be an object");
+				materializedRequest.manualInputs = materialized.value;
+			}
 			runs.set(runId, runState);
 			setLatestRunInfoMap({
 				latestRunInfoMap,
 				evalIds: getTargetEvalKeys({
-					request,
+					request: materializedRequest,
 					sortedEvals: getSortedEvalMetas()
 				}),
 				info: {
@@ -687,13 +1143,9 @@ function createRunner({ watchForChanges = true } = {}) {
 					evalSourceFingerprint: null
 				}
 			});
-			await mkdir(runDir, { recursive: true });
-			await mkdir(join(runDir, "traces"), { recursive: true });
-			await mkdir(join(runDir, "artifacts"), { recursive: true });
-			await mkdir(join(runDir, "case-details"), { recursive: true });
 			await writeFile(join(runDir, "run.json"), JSON.stringify(manifest, null, 2));
 			const childContext = {
-				request,
+				request: materializedRequest,
 				workspaceRoot,
 				runDir,
 				manifest,
@@ -776,18 +1228,7 @@ function createRunner({ watchForChanges = true } = {}) {
 			};
 		},
 		async close() {
-			if (discoveryRefreshTimer !== void 0) {
-				clearTimeout(discoveryRefreshTimer);
-				discoveryRefreshTimer = void 0;
-			}
-			if (runHistoryRefreshTimer !== void 0) {
-				clearTimeout(runHistoryRefreshTimer);
-				runHistoryRefreshTimer = void 0;
-			}
-			const watchers = [discoveryWatcher, runHistoryWatcher].filter((watcher) => watcher !== void 0);
-			discoveryWatcher = void 0;
-			runHistoryWatcher = void 0;
-			await Promise.all(watchers.map((watcher) => watcher.close()));
+			await Promise.all([closeWatchers(), configReload.close()]);
 		},
 		getWorkspaceRoot() {
 			return workspaceRoot;
@@ -805,6 +1246,39 @@ function createRunner({ watchForChanges = true } = {}) {
 			return resolveArtifactPath(join(localStateDir, "runs"), artifactId_);
 		}
 	};
+	async function loadRunnerState() {
+		config = await loadConfig();
+		workspaceRoot = config.workspaceRoot ?? process.cwd();
+		localStateDir = resolve(workspaceRoot, ".agent-evals");
+		llmCallsConfig = resolveLlmCallsConfig(config.llmCalls);
+		apiCallsConfig = resolveApiCallsConfig(config.apiCalls);
+		await mkdir(localStateDir, { recursive: true });
+		await mkdir(join(localStateDir, "runs"), { recursive: true });
+		await cleanupStagedManualInputFiles(workspaceRoot);
+		cacheStore = createFsCacheStore({
+			workspaceRoot,
+			dir: config.cache?.dir,
+			maxEntriesPerNamespace: config.cache?.maxEntriesPerNamespace ?? config.cache?.maxEntriesPerEval,
+			maxEntriesByNamespace: config.cache?.maxEntriesByNamespace
+		});
+		await loadPersistedRuns();
+		await runner.refreshDiscovery();
+		if (watchForChanges) await setupWatcher();
+	}
+	async function closeWatchers() {
+		if (discoveryRefreshTimer !== void 0) {
+			clearTimeout(discoveryRefreshTimer);
+			discoveryRefreshTimer = void 0;
+		}
+		if (runHistoryRefreshTimer !== void 0) {
+			clearTimeout(runHistoryRefreshTimer);
+			runHistoryRefreshTimer = void 0;
+		}
+		const watchers = [discoveryWatcher, runHistoryWatcher].filter((watcher) => watcher !== void 0);
+		discoveryWatcher = void 0;
+		runHistoryWatcher = void 0;
+		await Promise.all(watchers.map((watcher) => watcher.close()));
+	}
 	async function setupWatcher() {
 		const watcher = watch(getWatchRootsForIncludePatterns({
 			patterns: config.include,
@@ -829,7 +1303,7 @@ function createRunner({ watchForChanges = true } = {}) {
 		watcher.on("unlink", scheduleRefresh);
 		watcher.on("addDir", scheduleRefresh);
 		watcher.on("unlinkDir", scheduleRefresh);
-		await setupRunHistoryWatcher();
+		await Promise.all([setupRunHistoryWatcher(), configReload.setupWatcher()]);
 		await watcherReady;
 	}
 	async function setupRunHistoryWatcher() {
@@ -854,6 +1328,9 @@ function createRunner({ watchForChanges = true } = {}) {
 			watcher.once("ready", ready);
 		});
 	}
+	function getActiveRunCount() {
+		return [...runs.values()].filter((run) => run.manifest.status === "running").length;
+	}
 	function emitDiscoveryEvent() {
 		const lastRunStatuses = getLastRunStatuses({
 			runs: runs.values(),
@@ -873,6 +1350,10 @@ function createRunner({ watchForChanges = true } = {}) {
 			payload: runner.getEvals()
 		};
 		for (const listener of discoveryListeners) listener(event);
+		configReload.reloadIfPendingAndIdle();
+	}
+	function emitToDiscoveryListeners(event) {
+		for (const listener of discoveryListeners) listener(event);
 	}
 	function emitEvent(runState, event) {
 		for (const listener of runState.listeners) try {
@@ -915,6 +1396,345 @@ function createRunner({ watchForChanges = true } = {}) {
 	return runner;
 }
 //#endregion
+//#region src/cliHelp.ts
+/** Render the help block for a given CLI topic to stdout via `console.info`. */
+function printHelp(topic = "global") {
+	if (topic === "app") {
+		console.info(`
+agent-evals app - Start server with UI
+
+Usage:
+  agent-evals app [flags]
+
+Flags:
+  --port <n>                 Server port (default: 4100)
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "list") {
+		console.info(`
+agent-evals list - List discovered evals
+
+Usage:
+  agent-evals list [flags]
+
+Flags:
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "run") {
+		console.info(`
+agent-evals run - Run evals
+
+Usage:
+  agent-evals run [flags]
+
+Flags:
+  --eval <id>                Run specific eval(s) (comma-separated)
+  --file <path|glob>         Run eval files matching path/glob (comma-separated)
+  --case <id>                Run case(s); combine with --file/--eval if ambiguous
+  --trials <n>               Number of trials per case
+  --inspect[=host:port]      Run with the Node.js inspector enabled
+  --inspect-brk[=host:port]  Enable inspector and pause before startup
+  --json                     Output run summary as JSON
+  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
+  --no-cache                 Shortcut for --cache bypass
+  --refresh-cache            Shortcut for --cache refresh
+  --clear-cache              Clear the cache before starting the run
+  --input <json>             Manual input value for a single targeted eval
+                             that declares manualInput
+  --input-file <path>        JSON object keyed by eval key (or eval id) with
+                             manual input values for one or more targeted evals
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "show-runs") {
+		console.info(`
+agent-evals show-runs - Show saved run artifact file paths
+
+Usage:
+  agent-evals show-runs [<run-id>|latest] [--json]
+
+Prints the run directory and stable artifact paths for run.json, summary.json,
+cases.jsonl, case detail JSON, and trace JSON files. Run ids can be full
+timestamp ids, short ids such as r0, or latest.
+
+Flags:
+  --json                     Output the file index as JSON
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "cache" || topic === "cache list" || topic === "cache clear") {
+		console.info(`
+agent-evals cache - Manage cached operation entries
+
+Usage:
+  agent-evals cache list [flags]
+  agent-evals cache clear --eval <id>
+  agent-evals cache clear --all
+
+Flags:
+  --eval <id>                Clear entries for specific eval(s) (comma-separated)
+  --all                      Confirm clearing every cached entry
+  --json                     Output cache listing as JSON
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	console.info(`
+agent-evals - LLM/Agent eval runner
+
+Commands:
+  app                        Start server with UI
+  list                       List discovered evals
+  run                        Run evals
+  show-runs [id|latest]      Show saved run artifact file paths
+  cache list                 List cached operation entries
+  cache clear --eval <id>    Clear cache entries for one eval
+  cache clear --all          Clear every cached entry
+  help                       Show this help
+
+Options:
+  --eval <id>                Run specific eval(s) (comma-separated)
+  --case <id>                Run specific case(s) (comma-separated)
+  --trials <n>               Number of trials per case
+  --inspect[=host:port]      Run with the Node.js inspector enabled
+  --inspect-brk[=host:port]  Enable inspector and pause before startup
+  --json                     Output results as JSON
+  --port <n>                 Server port (default: 4100)
+  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
+  --no-cache                 Shortcut for --cache bypass
+  --refresh-cache            Shortcut for --cache refresh
+  --clear-cache              Clear the cache before starting the run
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show help
+  `);
+}
+//#endregion
+//#region src/manualInputArgs.ts
+function isPlainObject(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isPathInputObject(value) {
+	if (!isPlainObject(value)) return false;
+	return typeof value.path === "string" && (value.name === void 0 || typeof value.name === "string") && (value.mimeType === void 0 || typeof value.mimeType === "string");
+}
+function escapeRegex$1(value) {
+	return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+}
+function globToRegex$1(pattern) {
+	const normalized = pattern.replaceAll("\\", "/");
+	let regex = "^";
+	for (let i = 0; i < normalized.length; i++) {
+		const char = normalized[i];
+		const next = normalized[i + 1];
+		if (char === "*" && next === "*") {
+			regex += ".*";
+			i++;
+		} else if (char === "*") regex += "[^/]*";
+		else if (char === "?") regex += "[^/]";
+		else regex += escapeRegex$1(char ?? "");
+	}
+	regex += "$";
+	return new RegExp(regex);
+}
+function fileMatches$1(pattern, filePath) {
+	const normalizedPattern = pattern.replaceAll("\\", "/");
+	if (normalizedPattern === filePath) return true;
+	return globToRegex$1(normalizedPattern).test(filePath);
+}
+function isManualInputEvalTargeted(params) {
+	const { evalSummary, args } = params;
+	const hasEvalIds = args.evalIds.length > 0;
+	const hasFiles = args.files.length > 0;
+	const hasCaseIds = args.caseIds.length > 0;
+	if (hasEvalIds && !args.evalIds.includes(evalSummary.id)) return false;
+	if (hasFiles) {
+		if (!args.files.some((file) => fileMatches$1(file, evalSummary.filePath))) return false;
+	}
+	if (!hasEvalIds && !hasFiles) {
+		if (hasCaseIds) return false;
+		return true;
+	}
+	return true;
+}
+async function readInputFileMap(inputFilePath) {
+	const readResult = await resultify(() => readFile(inputFilePath, "utf-8"));
+	if (readResult.error) return {
+		error: `Failed to read --input-file at ${inputFilePath}: ${readResult.error.message}`,
+		value: null
+	};
+	const parseResult = resultify(() => JSON.parse(readResult.value));
+	if (parseResult.error) return {
+		error: `Failed to parse --input-file at ${inputFilePath} as JSON: ${parseResult.error.message}`,
+		value: null
+	};
+	return {
+		error: null,
+		value: parseResult.value
+	};
+}
+async function normalizeManualInputFileValue(params) {
+	if (isManualInputFileValue(params.value)) return {
+		error: null,
+		value: params.value
+	};
+	if (!isPathInputObject(params.value)) return {
+		error: null,
+		value: params.value
+	};
+	const pathInput = params.value;
+	const staged = await resultify(() => stageManualInputFileFromPath({
+		workspaceRoot: params.workspaceRoot,
+		path: pathInput.path,
+		name: pathInput.name,
+		mimeType: pathInput.mimeType
+	}));
+	if (staged.error) return {
+		error: `Failed to stage file input "${params.fieldKey}" for eval "${params.evalId}": ${staged.error.message}`,
+		value: null
+	};
+	return {
+		error: null,
+		value: staged.value
+	};
+}
+async function normalizeManualInputValue(params) {
+	const descriptor = params.evalSummary.manualInput;
+	if (!descriptor || !isPlainObject(params.value)) return {
+		error: null,
+		value: params.value
+	};
+	const next = { ...params.value };
+	for (const field of descriptor.fields) {
+		if (field.kind !== "file") continue;
+		const normalized = await normalizeManualInputFileValue({
+			workspaceRoot: params.workspaceRoot,
+			evalId: params.evalSummary.id,
+			fieldKey: field.key,
+			value: next[field.key]
+		});
+		if (normalized.error !== null) return {
+			error: normalized.error,
+			value: null
+		};
+		next[field.key] = normalized.value;
+	}
+	return {
+		error: null,
+		value: next
+	};
+}
+/**
+* Resolve the `manualInputs` payload to send with `runner.startRun`.
+*
+* Inspects every discovered eval that declares `manualInput`, filters them to
+* the run target, and either returns the typed map (single eval via `--input`,
+* multiple via `--input-file`) or a structured error to display and exit on.
+*/
+async function collectManualInputs(params) {
+	const { runner, args } = params;
+	const workspaceRoot = runner.getWorkspaceRoot();
+	const targetedManualInputEvals = runner.getEvals().filter((evalSummary) => evalSummary.manualInput !== void 0).filter((evalSummary) => isManualInputEvalTargeted({
+		evalSummary,
+		args
+	}));
+	if (targetedManualInputEvals.length === 0) {
+		if (args.inputJson !== void 0 || args.inputFilePath !== void 0) return {
+			error: "--input/--input-file was provided but no targeted eval requires manual input.",
+			value: null
+		};
+		return {
+			error: null,
+			value: void 0
+		};
+	}
+	if (args.inputJson !== void 0 && args.inputFilePath !== void 0) return {
+		error: "Cannot use --input and --input-file together; choose one.",
+		value: null
+	};
+	if (args.inputJson !== void 0) {
+		if (targetedManualInputEvals.length > 1) {
+			const ids = targetedManualInputEvals.map((evalSummary) => evalSummary.id).join(", ");
+			return {
+				error: `--input only works for one targeted manual-input eval at a time; got ${String(targetedManualInputEvals.length)} (${ids}). Use --input-file with a JSON object keyed by eval key.`,
+				value: null
+			};
+		}
+		const parsedResult = resultify(() => JSON.parse(args.inputJson ?? ""));
+		if (parsedResult.error) return {
+			error: `Failed to parse --input as JSON: ${parsedResult.error.message}`,
+			value: null
+		};
+		const [onlyEval] = targetedManualInputEvals;
+		if (onlyEval === void 0) return {
+			error: null,
+			value: void 0
+		};
+		const normalized = await normalizeManualInputValue({
+			workspaceRoot,
+			evalSummary: onlyEval,
+			value: parsedResult.value
+		});
+		if (normalized.error !== null) return {
+			error: normalized.error,
+			value: null
+		};
+		return {
+			error: null,
+			value: { [onlyEval.key]: normalized.value }
+		};
+	}
+	if (args.inputFilePath !== void 0) {
+		const fileResult = await readInputFileMap(args.inputFilePath);
+		if (fileResult.error !== null) return {
+			error: fileResult.error,
+			value: null
+		};
+		if (!isPlainObject(fileResult.value)) return {
+			error: `--input-file must contain a JSON object keyed by eval key (got ${typeof fileResult.value}).`,
+			value: null
+		};
+		const map = {};
+		for (const evalSummary of targetedManualInputEvals) {
+			const byKey = fileResult.value[evalSummary.key];
+			const byId = fileResult.value[evalSummary.id];
+			const value = byKey !== void 0 ? byKey : byId;
+			if (value === void 0) return {
+				error: `--input-file is missing manual input for eval "${evalSummary.id}" (key "${evalSummary.key}").`,
+				value: null
+			};
+			const normalized = await normalizeManualInputValue({
+				workspaceRoot,
+				evalSummary,
+				value
+			});
+			if (normalized.error !== null) return {
+				error: normalized.error,
+				value: null
+			};
+			map[evalSummary.key] = normalized.value;
+		}
+		return {
+			error: null,
+			value: map
+		};
+	}
+	return {
+		error: `Eval(s) require manual input but no --input/--input-file was provided: ${targetedManualInputEvals.map((evalSummary) => evalSummary.id).join(", ")}`,
+		value: null
+	};
+}
+//#endregion
 //#region src/cli.ts
 function parseArgs(argv) {
 	const normalizedArgv = argv.filter((arg) => arg !== "--no-env");
@@ -934,7 +1754,9 @@ function parseArgs(argv) {
 		cacheMode: "use",
 		clearCache: false,
 		all: false,
-		loadEnv: normalizedArgv.length === argv.length
+		loadEnv: normalizedArgv.length === argv.length,
+		inputJson: void 0,
+		inputFilePath: void 0
 	};
 	const command = normalizedArgv[0];
 	if (command === "--help" || command === "-h") {
@@ -981,7 +1803,13 @@ function parseArgs(argv) {
 		} else if (arg === "--no-cache") args.cacheMode = "bypass";
 		else if (arg === "--refresh-cache") args.cacheMode = "refresh";
 		else if (arg === "--clear-cache") args.clearCache = true;
-		else if (arg === "--all") args.all = true;
+		else if (arg === "--input" && next !== void 0) {
+			args.inputJson = next;
+			i++;
+		} else if (arg === "--input-file" && next !== void 0) {
+			args.inputFilePath = next;
+			i++;
+		} else if (arg === "--all") args.all = true;
 		else if (!arg.startsWith("-")) args.positionals.push(arg);
 	}
 	return args;
@@ -1112,8 +1940,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-mBbAN-Gt.mjs");
-	const runnerModule = await import("./runner-BQn_xf36.mjs");
+	const appModule = await import("./app-D6-msfKP.mjs");
+	const runnerModule = await import("./runner-Bq1f9B9d.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();
@@ -1184,10 +2012,26 @@ async function commandRun(args) {
 		mode: "evalIds",
 		files: args.files
 	} : { mode: "all" };
+	const manualInputsResult = await collectManualInputs({
+		runner,
+		args: {
+			evalIds: args.evalIds,
+			files: args.files,
+			caseIds: args.caseIds,
+			inputJson: args.inputJson,
+			inputFilePath: args.inputFilePath
+		}
+	});
+	if (manualInputsResult.error !== null) {
+		console.error(manualInputsResult.error);
+		process.exit(1);
+		return;
+	}
 	const run = await runner.startRun({
 		target,
 		trials: args.trials,
-		cache: { mode: args.cacheMode }
+		cache: { mode: args.cacheMode },
+		manualInputs: manualInputsResult.value
 	});
 	if (!args.json) {
 		console.info(`Run started: ${run.manifest.id}`);
@@ -1405,122 +2249,5 @@ async function waitForRunCompletion(runner, runId) {
 		check();
 	});
 }
-function printHelp(topic = "global") {
-	if (topic === "app") {
-		console.info(`
-agent-evals app - Start server with UI
-
-Usage:
-  agent-evals app [flags]
-
-Flags:
-  --port <n>                 Server port (default: 4100)
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	if (topic === "list") {
-		console.info(`
-agent-evals list - List discovered evals
-
-Usage:
-  agent-evals list [flags]
-
-Flags:
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	if (topic === "run") {
-		console.info(`
-agent-evals run - Run evals
-
-Usage:
-  agent-evals run [flags]
-
-Flags:
-  --eval <id>                Run specific eval(s) (comma-separated)
-  --file <path|glob>         Run eval files matching path/glob (comma-separated)
-  --case <id>                Run case(s); combine with --file/--eval if ambiguous
-  --trials <n>               Number of trials per case
-  --inspect[=host:port]      Run with the Node.js inspector enabled
-  --inspect-brk[=host:port]  Enable inspector and pause before startup
-  --json                     Output run summary as JSON
-  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
-  --no-cache                 Shortcut for --cache bypass
-  --refresh-cache            Shortcut for --cache refresh
-  --clear-cache              Clear the cache before starting the run
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	if (topic === "show-runs") {
-		console.info(`
-agent-evals show-runs - Show saved run artifact file paths
-
-Usage:
-  agent-evals show-runs [<run-id>|latest] [--json]
-
-Prints the run directory and stable artifact paths for run.json, summary.json,
-cases.jsonl, case detail JSON, and trace JSON files. Run ids can be full
-timestamp ids, short ids such as r0, or latest.
-
-Flags:
-  --json                     Output the file index as JSON
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	if (topic === "cache" || topic === "cache list" || topic === "cache clear") {
-		console.info(`
-agent-evals cache - Manage cached operation entries
-
-Usage:
-  agent-evals cache list [flags]
-  agent-evals cache clear --eval <id>
-  agent-evals cache clear --all
-
-Flags:
-  --eval <id>                Clear entries for specific eval(s) (comma-separated)
-  --all                      Confirm clearing every cached entry
-  --json                     Output cache listing as JSON
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	console.info(`
-agent-evals - LLM/Agent eval runner
-
-Commands:
-  app                        Start server with UI
-  list                       List discovered evals
-  run                        Run evals
-  show-runs [id|latest]      Show saved run artifact file paths
-  cache list                 List cached operation entries
-  cache clear --eval <id>    Clear cache entries for one eval
-  cache clear --all          Clear every cached entry
-  help                       Show this help
-
-Options:
-  --eval <id>                Run specific eval(s) (comma-separated)
-  --case <id>                Run specific case(s) (comma-separated)
-  --trials <n>               Number of trials per case
-  --inspect[=host:port]      Run with the Node.js inspector enabled
-  --inspect-brk[=host:port]  Enable inspector and pause before startup
-  --json                     Output results as JSON
-  --port <n>                 Server port (default: 4100)
-  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
-  --no-cache                 Shortcut for --cache bypass
-  --refresh-cache            Shortcut for --cache refresh
-  --clear-cache              Clear the cache before starting the run
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show help
-  `);
-}
 //#endregion
-export { createRunner as n, runCli as t };
+export { materializeManualInputFiles as a, isManualInputFileValue as i, createRunner as n, stageManualInputFile as o, cleanupStagedManualInputFiles as r, stageManualInputFileFromPath as s, runCli as t };