@ls-stack/agent-eval 0.2.0 → 0.3.0

package/dist/{cli-CwEFLP0w.mjs → cli-Dw9et3_Z.mjs}

@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
 import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import { dirname, extname, join, relative, resolve } from "node:path";
 import { AsyncLocalStorage } from "node:async_hooks";
+import { Buffer as Buffer$1 } from "node:buffer";
 import { getCompositeKey } from "@ls-stack/utils/getCompositeKey";
 import { z } from "zod/v4";
 import { watch } from "chokidar";
@@ -206,6 +207,15 @@ function noopActiveSpan() {
 		setAttributes() {}
 	};
 }
+function noopExternalSpan(id) {
+	return {
+		id,
+		setName() {},
+		setAttribute() {},
+		setAttributes() {},
+		end() {}
+	};
+}
 function mergeSpanAttributes(span, attributes) {
 	span.attributes = {
 		...span.attributes,
@@ -225,6 +235,127 @@ function createSpanHandle(span) {
 		}
 	};
 }
+function createExternalSpanHandle(id) {
+	return {
+		id,
+		setName(value) {
+			updateExternalSpan({
+				id,
+				name: value
+			});
+		},
+		setAttribute(key, value) {
+			updateExternalSpan({
+				id,
+				attributes: { [key]: value }
+			});
+		},
+		setAttributes(value) {
+			updateExternalSpan({
+				id,
+				attributes: value
+			});
+		},
+		end(info = {}) {
+			endExternalSpan({
+				...info,
+				id
+			});
+		}
+	};
+}
+function toIsoTimestamp(value) {
+	if (value === void 0) return (/* @__PURE__ */ new Date()).toISOString();
+	if (typeof value === "string") return value;
+	return value.toISOString();
+}
+function findSpan(scope, id) {
+	return scope.spans.find((span) => span.id === id);
+}
+function resolveExternalParentId(scope, parentId) {
+	if (parentId !== void 0) return parentId;
+	return scope.activeSpanStack.at(-1)?.id ?? null;
+}
+function startExternalSpan(info) {
+	const id = info.id ?? generateSpanId();
+	const scope = getCurrentScope();
+	if (!scope) return noopExternalSpan(id);
+	const existing = findSpan(scope, id);
+	if (existing) {
+		existing.parentId = resolveExternalParentId(scope, info.parentId);
+		existing.kind = info.kind;
+		existing.name = info.name;
+		existing.startedAt = toIsoTimestamp(info.startedAt);
+		existing.status = "running";
+		existing.endedAt = null;
+		if (info.attributes !== void 0) existing.attributes = info.attributes;
+		return createExternalSpanHandle(id);
+	}
+	scope.spans.push({
+		id,
+		parentId: resolveExternalParentId(scope, info.parentId),
+		caseId: scope.caseId,
+		kind: info.kind,
+		name: info.name,
+		startedAt: toIsoTimestamp(info.startedAt),
+		endedAt: null,
+		status: "running",
+		attributes: info.attributes
+	});
+	return createExternalSpanHandle(id);
+}
+function updateExternalSpan(info) {
+	const scope = getCurrentScope();
+	if (!scope) return;
+	const span = findSpan(scope, info.id);
+	if (!span) return;
+	if (info.name !== void 0) span.name = info.name;
+	if (info.status !== void 0) span.status = info.status;
+	if (info.error !== void 0) span.error = info.error;
+	if (info.attributes !== void 0) mergeSpanAttributes(span, info.attributes);
+}
+function endExternalSpan(info) {
+	const scope = getCurrentScope();
+	if (!scope) return;
+	const span = findSpan(scope, info.id);
+	if (!span) return;
+	updateExternalSpan(info);
+	span.status = info.status ?? (info.error ? "error" : "ok");
+	span.endedAt = toIsoTimestamp(info.endedAt);
+}
+function recordExternalSpan(info) {
+	const id = info.id ?? generateSpanId();
+	const scope = getCurrentScope();
+	if (!scope) return id;
+	const startedAt = toIsoTimestamp(info.startedAt);
+	const endedAt = info.endedAt === null ? null : info.endedAt ? toIsoTimestamp(info.endedAt) : startedAt;
+	const existing = findSpan(scope, id);
+	const status = info.status ?? (info.error ? "error" : "ok");
+	if (existing) {
+		existing.parentId = resolveExternalParentId(scope, info.parentId);
+		existing.kind = info.kind;
+		existing.name = info.name;
+		existing.startedAt = startedAt;
+		existing.endedAt = endedAt;
+		existing.status = status;
+		existing.attributes = info.attributes;
+		existing.error = info.error;
+		return id;
+	}
+	scope.spans.push({
+		id,
+		parentId: resolveExternalParentId(scope, info.parentId),
+		caseId: scope.caseId,
+		kind: info.kind,
+		name: info.name,
+		startedAt,
+		endedAt,
+		status,
+		attributes: info.attributes,
+		error: info.error
+	});
+	return id;
+}
 /**
 * Ambient handle for the active span in the current async context.
 *
@@ -272,7 +403,7 @@ async function traceSpan(info, fn) {
 		if (cacheOpts !== void 0 && cacheCtx !== void 0 && scope.replayingDepth === 0) {
 			const ctx = cacheCtx;
 			const namespace = cacheOpts.namespace ?? `${ctx.evalId}__${info.name}`;
-			const keyHash = hashCacheKey({
+			const keyHash = await hashCacheKey({
 				namespace,
 				codeFingerprint: ctx.codeFingerprint,
 				key: cacheOpts.key
@@ -359,6 +490,34 @@ async function traceSpan(info, fn) {
 const evalTracer = {
 	/** Run a callback inside a new trace span and record its lifecycle. */
 	span: traceSpan,
+	/**
+	* Start a span whose lifecycle is controlled by an external tracer/exporter.
+	*
+	* Calls are no-ops outside an eval case scope, except that a generated or
+	* caller-provided id is still returned for ergonomic adapter code.
+	*/
+	startSpan: startExternalSpan,
+	/**
+	* Merge updates into an externally managed span that was started earlier.
+	*
+	* This is intended for observability exporters that receive span update
+	* events before the final end event.
+	*/
+	updateSpan: updateExternalSpan,
+	/**
+	* Finish an externally managed span and attach final attributes or errors.
+	*
+	* Missing spans are ignored so exporter adapters can safely forward events
+	* even when they are emitted outside an eval case scope.
+	*/
+	endSpan: endExternalSpan,
+	/**
+	* Record a complete external span in one call.
+	*
+	* Use this when an upstream tracer only exposes completed spans rather than
+	* start/update/end events.
+	*/
+	recordSpan: recordExternalSpan,
 	/** Record a named point-in-time value alongside the trace. */
 	checkpoint(name, data) {
 		const scope = getCurrentScope();
@@ -412,9 +571,103 @@ function buildTraceTree(spans, checkpoints) {
 		checkpoints
 	};
 }
-/** Hash the components of a cache key into a deterministic hex digest. */
-function hashCacheKey(input) {
-	return createHash("sha256").update(getCompositeKey(input)).digest("hex");
+var SerializedCacheKeyValue = class {
+	value;
+	constructor(value) {
+		this.value = value;
+	}
+};
+/**
+* Hash the components of a cache key into a deterministic hex digest.
+*
+* Native `Blob` and `File` values are read asynchronously and hashed by
+* content. Use `hashCacheKeySync` only when the key contains no async values.
+*/
+async function hashCacheKey(input) {
+	return hashCacheKeySyncMaterialized(await materializeAsyncCacheKeyValue(input));
+}
+/**
+* Synchronously hash cache key components. This supports JSON-like data and
+* in-memory binary values such as `Buffer`, `ArrayBuffer`, and typed arrays,
+* but cannot content-hash native `Blob` or `File` values.
+*/
+function hashCacheKeySync(input) {
+	return hashCacheKeySyncMaterialized(input);
+}
+function hashCacheKeySyncMaterialized(input) {
+	return createHash("sha256").update(getCompositeKey(input, { stringify: stringifyCacheKeyValue })).digest("hex");
+}
+function stringifyCacheKeyValue(value) {
+	if (value instanceof SerializedCacheKeyValue) return value.value;
+	if (Buffer$1.isBuffer(value)) return `$buffer:${hashBytes(value)}`;
+	if (isArrayBuffer(value)) return `$arrayBuffer:${hashBytes(new Uint8Array(value))}`;
+	if (isSharedArrayBuffer(value)) return `$sharedArrayBuffer:${hashBytes(new Uint8Array(value))}`;
+	if (isArrayBufferView(value)) {
+		const bytes = new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+		return `$${value.constructor.name}:${hashBytes(bytes)}`;
+	}
+	if (isFile$1(value)) return `$file:${getCompositeKey({
+		lastModified: value.lastModified,
+		name: value.name,
+		size: value.size,
+		type: value.type
+	})}`;
+	if (isBlob$1(value)) return `$blob:${getCompositeKey({
+		size: value.size,
+		type: value.type
+	})}`;
+}
+async function materializeAsyncCacheKeyValue(value, refs = /* @__PURE__ */ new WeakSet()) {
+	const serialized = await stringifyAsyncCacheKeyValue(value);
+	if (serialized !== void 0) return new SerializedCacheKeyValue(serialized);
+	if (stringifyCacheKeyValue(value) !== void 0) return value;
+	if (!value || typeof value !== "object") return value;
+	if (Array.isArray(value)) {
+		const items = [];
+		for (const item of value) items.push(await materializeAsyncCacheKeyValue(item, refs));
+		return items;
+	}
+	if (refs.has(value)) throw new Error("Circular reference detected");
+	refs.add(value);
+	const entries = [];
+	for (const [key, entryValue] of Object.entries(value)) entries.push([key, await materializeAsyncCacheKeyValue(entryValue, refs)]);
+	refs.delete(value);
+	return Object.fromEntries(entries);
+}
+async function stringifyAsyncCacheKeyValue(value) {
+	if (isFile$1(value)) return `$file:${getCompositeKey({
+		bytes: await hashBlobBytes(value),
+		lastModified: value.lastModified,
+		name: value.name,
+		size: value.size,
+		type: value.type
+	})}`;
+	if (isBlob$1(value)) return `$blob:${getCompositeKey({
+		bytes: await hashBlobBytes(value),
+		size: value.size,
+		type: value.type
+	})}`;
+}
+async function hashBlobBytes(value) {
+	return hashBytes(new Uint8Array(await value.arrayBuffer()));
+}
+function hashBytes(value) {
+	return createHash("sha256").update(value).digest("hex");
+}
+function isArrayBuffer(value) {
+	return value instanceof ArrayBuffer;
+}
+function isSharedArrayBuffer(value) {
+	return value instanceof SharedArrayBuffer;
+}
+function isArrayBufferView(value) {
+	return ArrayBuffer.isView(value);
+}
+function isBlob$1(value) {
+	return value instanceof Blob;
+}
+function isFile$1(value) {
+	return value instanceof File;
 }
 function toJsonSafe(value) {
 	if (value === void 0) return void 0;
@@ -989,6 +1242,12 @@ const cacheEntrySchema = z.object({
 	codeFingerprint: z.string(),
 	recording: cacheRecordingSchema
 });
+/** Persisted per-owner cache file containing multiple cache entries. */
+const cacheFileSchema = z.object({
+	version: z.literal(1),
+	owner: z.string(),
+	entries: z.record(z.string(), cacheEntrySchema)
+});
 //#endregion
 //#region ../shared/src/schemas/config.ts
 /** Strategy used to collapse repeated trials into one stored case result. */
@@ -1004,7 +1263,8 @@ const agentEvalsConfigSchema = z.object({
 	traceDisplay: traceDisplayInputConfigSchema.optional(),
 	cache: z.object({
 		enabled: z.boolean().optional(),
-		dir: z.string().optional()
+		dir: z.string().optional(),
+		maxEntriesPerEval: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional())
 	}).optional()
 });
 //#endregion
@@ -1243,60 +1503,59 @@ const createRunRequestSchema = z.object({
 const updateManualScoreRequestSchema = z.object({ value: z.number().min(0).max(1).nullable() });
 //#endregion
 //#region ../runner/src/cacheStore.ts
+const defaultMaxEntriesPerEval = 100;
 /**
 * Create a filesystem-backed cache adapter rooted at `<workspaceRoot>/<dir>`.
 *
-* Writes use `<name>.tmp` + atomic `rename` to avoid partial reads under
-* concurrent access.
+* Cache entries are grouped into one inspectable JSON file per eval/cache
+* owner. Writes use a short-lived lock directory plus `<name>.tmp` + atomic
+* `rename` to avoid partial reads and lost updates under concurrent access.
 */
 function createFsCacheStore(options) {
 	const cacheDir = resolve(options.workspaceRoot, options.dir ?? ".agent-evals/cache");
+	const maxEntriesPerEval = normalizeMaxEntries(options.maxEntriesPerEval);
 	return {
 		dir() {
 			return cacheDir;
 		},
 		async lookup(namespace, keyHash) {
-			const filePath = entryPath(cacheDir, namespace, keyHash);
-			if (!existsSync(filePath)) return null;
-			const json = safeJsonParse(await readFile(filePath, "utf-8"));
-			if (json === null) return null;
-			const parsed = cacheEntrySchema.safeParse(json);
-			if (!parsed.success) return null;
-			return parsed.data;
+			return (await readCacheFile(cacheDir, ownerFromNamespace(namespace)))?.entries[keyHash] ?? null;
 		},
 		async write(entry) {
-			const filePath = entryPath(cacheDir, entry.namespace, entry.key);
-			await mkdir(dirname(filePath), { recursive: true });
-			const tmpPath = `${filePath}.${process.pid.toString()}.tmp`;
-			await writeFile(tmpPath, JSON.stringify(entry));
-			await rename(tmpPath, filePath);
+			const owner = ownerFromNamespace(entry.namespace);
+			const filePath = ownerPath(cacheDir, owner);
+			await mkdir(cacheDir, { recursive: true });
+			await withCacheFileLock(filePath, async () => {
+				await writeCacheFile(cacheDir, {
+					version: 1,
+					owner,
+					entries: pruneEntries({
+						...(await readCacheFile(cacheDir, owner))?.entries ?? {},
+						[entry.key]: entry
+					}, maxEntriesPerEval, entry.key)
+				});
+			});
 		},
 		async list() {
 			if (!existsSync(cacheDir)) return [];
-			const namespaces = await readdir(cacheDir);
+			const files = await readdir(cacheDir);
 			const items = [];
-			for (const namespace of namespaces) {
-				const nsPath = join(cacheDir, namespace);
-				if (!(await stat(nsPath)).isDirectory()) continue;
-				const files = await readdir(nsPath);
-				for (const fileName of files) {
-					if (!fileName.endsWith(".json")) continue;
-					const filePath = join(nsPath, fileName);
-					const json = safeJsonParse(await readFile(filePath, "utf-8"));
-					if (json === null) continue;
-					const parsed = cacheEntrySchema.safeParse(json);
-					if (!parsed.success) continue;
-					const fileStat = await stat(filePath);
-					items.push({
-						key: parsed.data.key,
-						namespace: parsed.data.namespace,
-						spanName: parsed.data.spanName,
-						spanKind: parsed.data.spanKind,
-						storedAt: parsed.data.storedAt,
-						codeFingerprint: parsed.data.codeFingerprint,
-						sizeBytes: fileStat.size
-					});
-				}
+			for (const fileName of files) {
+				if (!fileName.endsWith(".json")) continue;
+				const filePath = join(cacheDir, fileName);
+				const fileStatResult = await resultify(() => stat(filePath));
+				if (fileStatResult.error || !fileStatResult.value.isFile()) continue;
+				const cacheFile = await readCacheFilePath(filePath);
+				if (cacheFile === null) continue;
+				for (const entry of Object.values(cacheFile.entries)) items.push({
+					key: entry.key,
+					namespace: entry.namespace,
+					spanName: entry.spanName,
+					spanKind: entry.spanKind,
+					storedAt: entry.storedAt,
+					codeFingerprint: entry.codeFingerprint,
+					sizeBytes: Buffer.byteLength(JSON.stringify(entry), "utf8")
+				});
 			}
 			items.sort((a, b) => a.storedAt < b.storedAt ? 1 : -1);
 			return items;
@@ -1310,21 +1569,36 @@ function createFsCacheStore(options) {
 				});
 				return;
 			}
-			if (filter.namespace !== void 0 && filter.key === void 0) {
-				await rm(join(cacheDir, filter.namespace), {
-					recursive: true,
-					force: true
+			if (filter.namespace !== void 0) {
+				const owner = ownerFromNamespace(filter.namespace);
+				await withCacheFileLock(ownerPath(cacheDir, owner), async () => {
+					const cacheFile = await readCacheFile(cacheDir, owner);
+					if (cacheFile === null) return;
+					await writeOrRemoveCacheFile(cacheDir, {
+						version: 1,
+						owner,
+						entries: Object.fromEntries(Object.entries(cacheFile.entries).filter(([key, entry]) => {
+							if (filter.key !== void 0) return key !== filter.key;
+							return entry.namespace !== filter.namespace;
+						}))
+					});
 				});
 				return;
 			}
-			if (filter.namespace !== void 0 && filter.key !== void 0) {
-				await rm(entryPath(cacheDir, filter.namespace, filter.key), { force: true });
-				return;
-			}
-			const namespaces = await readdir(cacheDir);
-			for (const namespace of namespaces) {
-				const filePath = entryPath(cacheDir, namespace, filter.key ?? "");
-				if (existsSync(filePath)) await rm(filePath, { force: true });
+			const files = await readdir(cacheDir);
+			for (const fileName of files) {
+				if (!fileName.endsWith(".json")) continue;
+				const filePath = join(cacheDir, fileName);
+				await withCacheFileLock(filePath, async () => {
+					const cacheFile = await readCacheFilePath(filePath);
+					if (cacheFile === null) return;
+					const entries = Object.fromEntries(Object.entries(cacheFile.entries).filter(([key]) => key !== filter.key));
+					await writeOrRemoveCacheFile(cacheDir, {
+						version: 1,
+						owner: cacheFile.owner,
+						entries
+					});
+				});
 			}
 		}
 	};
@@ -1356,8 +1630,16 @@ function createBufferedCacheStore(backingStore) {
 		}
 	};
 }
-function entryPath(cacheDir, namespace, keyHash) {
-	return join(cacheDir, sanitizeSegment$1(namespace), `${keyHash}.json`);
+function normalizeMaxEntries(value) {
+	if (value === void 0 || !Number.isFinite(value) || value <= 0) return defaultMaxEntriesPerEval;
+	return Math.floor(value);
+}
+function ownerFromNamespace(namespace) {
+	const [owner] = namespace.split("__");
+	return owner === void 0 || owner.length === 0 ? namespace : owner;
+}
+function ownerPath(cacheDir, owner) {
+	return join(cacheDir, `${sanitizeSegment$1(owner)}.json`);
 }
 function toPendingKey(namespace, keyHash) {
 	return `${namespace}::${keyHash}`;
@@ -1365,6 +1647,69 @@ function toPendingKey(namespace, keyHash) {
 function sanitizeSegment$1(segment) {
 	return segment.replace(/[^a-zA-Z0-9_.-]/g, "_");
 }
+async function readCacheFile(cacheDir, owner) {
+	return readCacheFilePath(ownerPath(cacheDir, owner));
+}
+async function readCacheFilePath(filePath) {
+	if (!existsSync(filePath)) return null;
+	const rawResult = await resultify(() => readFile(filePath, "utf-8"));
+	if (rawResult.error) return null;
+	const json = safeJsonParse(rawResult.value);
+	if (json === null) return null;
+	const parsed = cacheFileSchema.safeParse(json);
+	if (!parsed.success) return null;
+	return parsed.data;
+}
+async function writeOrRemoveCacheFile(cacheDir, cacheFile) {
+	if (Object.keys(cacheFile.entries).length === 0) {
+		await rm(ownerPath(cacheDir, cacheFile.owner), { force: true });
+		return;
+	}
+	await writeCacheFile(cacheDir, cacheFile);
+}
+async function writeCacheFile(cacheDir, cacheFile) {
+	await mkdir(cacheDir, { recursive: true });
+	const filePath = ownerPath(cacheDir, cacheFile.owner);
+	const tmpPath = `${filePath}.${process.pid.toString()}.tmp`;
+	await writeFile(tmpPath, JSON.stringify(cacheFile, null, 2));
+	await rename(tmpPath, filePath);
+}
+function pruneEntries(entries, maxEntries, protectedKey) {
+	const sorted = Object.values(entries).toSorted((a, b) => a.storedAt < b.storedAt ? 1 : -1);
+	const kept = /* @__PURE__ */ new Map();
+	const protectedEntry = entries[protectedKey];
+	if (protectedEntry !== void 0) kept.set(protectedEntry.key, protectedEntry);
+	for (const entry of sorted) {
+		if (kept.size >= maxEntries) break;
+		kept.set(entry.key, entry);
+	}
+	return Object.fromEntries([...kept.values()].toSorted((a, b) => a.key < b.key ? -1 : 1).map((entry) => [entry.key, entry]));
+}
+async function withCacheFileLock(filePath, fn) {
+	const lockPath = `${filePath}.lock`;
+	await acquireLock(lockPath);
+	const result = await resultify(fn);
+	await rm(lockPath, {
+		recursive: true,
+		force: true
+	});
+	if (result.error) throw result.error;
+}
+async function acquireLock(lockPath) {
+	const startedAt = Date.now();
+	let lastError;
+	while (Date.now() - startedAt < 5e3) {
+		const result = await resultify(() => mkdir(lockPath, { recursive: false }));
+		if (!result.error) return;
+		lastError = result.error;
+		await sleep(20);
+	}
+	if (lastError instanceof Error) throw lastError;
+	throw new Error(`Timed out acquiring cache lock at ${lockPath}`);
+}
+function sleep(ms) {
+	return new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
+}
 function safeJsonParse(text) {
 	const parsed = resultify(() => JSON.parse(text));
 	if (parsed.error) return null;
@@ -2730,7 +3075,8 @@ function createRunner({ watchForChanges = true } = {}) {
 			await mkdir(join(localStateDir, "runs"), { recursive: true });
 			cacheStore = createFsCacheStore({
 				workspaceRoot,
-				dir: config.cache?.dir
+				dir: config.cache?.dir,
+				maxEntriesPerEval: config.cache?.maxEntriesPerEval
 			});
 			await loadPersistedRuns();
 			await runner.refreshDiscovery();
@@ -3128,6 +3474,9 @@ function parseArgs(argv) {
 	const args = {
 		command: "help",
 		subcommand: void 0,
+		showHelp: false,
+		helpTopic: "global",
+		unknownHelpTarget: void 0,
 		evalIds: [],
 		caseIds: [],
 		trials: 1,
@@ -3138,19 +3487,28 @@ function parseArgs(argv) {
 		all: false
 	};
 	const command = argv[0];
-	if (command === "app" || command === "list" || command === "run" || command === "cache" || command === "help") args.command = command;
+	if (command === "--help" || command === "-h") {
+		args.showHelp = true;
+		return args;
+	}
+	if (isCliCommand(command)) {
+		args.command = command;
+		args.helpTopic = command === "help" ? "global" : command;
+	} else if (command !== void 0 && !command.startsWith("-")) args.unknownHelpTarget = command;
 	let cursor = 1;
 	if (args.command === "cache") {
 		const sub = argv[cursor];
 		if (sub === "list" || sub === "clear") {
 			args.subcommand = sub;
+			args.helpTopic = `cache ${sub}`;
 			cursor++;
-		}
+		} else if (sub !== void 0 && !sub.startsWith("-")) args.unknownHelpTarget = `cache ${sub}`;
 	}
 	for (let i = cursor; i < argv.length; i++) {
 		const arg = argv[i];
 		const next = argv[i + 1];
-		if (arg === "--eval" && next) {
+		if (arg === "--help" || arg === "-h") args.showHelp = true;
+		else if (arg === "--eval" && next) {
 			args.evalIds.push(...next.split(","));
 			i++;
 		} else if (arg === "--case" && next) {
@@ -3180,6 +3538,15 @@ function parseArgs(argv) {
 */
 async function runCli(argv) {
 	const args = parseArgs(argv);
+	if (args.showHelp) {
+		if (args.unknownHelpTarget !== void 0) {
+			console.error(`No help found for "${args.unknownHelpTarget}".`);
+			process.exit(1);
+			return;
+		}
+		printHelp(args.helpTopic);
+		return;
+	}
 	switch (args.command) {
 		case "app":
 			await commandApp(args);
@@ -3194,10 +3561,13 @@ async function runCli(argv) {
 			await commandCache(args);
 			break;
 		default:
-			printHelp();
+			printHelp(args.helpTopic);
 			break;
 	}
 }
+function isCliCommand(command) {
+	return command === "app" || command === "list" || command === "run" || command === "cache" || command === "help";
+}
 const currentDir = dirname(fileURLToPath(import.meta.url));
 const repoRoot = resolve(currentDir, "../../..");
 const pnpmCommand = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
@@ -3246,8 +3616,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-CKa9TjXw.mjs");
-	const runnerModule = await import("./runner-Ck4X0H3p.mjs");
+	const appModule = await import("./app-DXYLqlWb.mjs");
+	const runnerModule = await import("./runner-CToL8eJs.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();
@@ -3378,7 +3748,7 @@ async function commandCache(args) {
 		process.exit(1);
 		return;
 	}
-	printHelp();
+	printHelp(args.helpTopic);
 }
 async function waitForRunCompletion(runner, runId) {
 	return new Promise((resolvePromise) => {
@@ -3393,7 +3763,69 @@ async function waitForRunCompletion(runner, runId) {
 		check();
 	});
 }
-function printHelp() {
+function printHelp(topic = "global") {
+	if (topic === "app") {
+		console.info(`
+agent-evals app - Start server with UI
+
+Usage:
+  agent-evals app [flags]
+
+Flags:
+  --port <n>                 Server port (default: 4100)
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "list") {
+		console.info(`
+agent-evals list - List discovered evals
+
+Usage:
+  agent-evals list [flags]
+
+Flags:
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "run") {
+		console.info(`
+agent-evals run - Run evals
+
+Usage:
+  agent-evals run [flags]
+
+Flags:
+  --eval <id>                Run specific eval(s) (comma-separated)
+  --case <id>                Run specific case(s) (comma-separated)
+  --trials <n>               Number of trials per case
+  --json                     Output run summary as JSON
+  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
+  --no-cache                 Shortcut for --cache bypass
+  --refresh-cache            Shortcut for --cache refresh
+  --clear-cache              Clear the cache before starting the run
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "cache" || topic === "cache list" || topic === "cache clear") {
+		console.info(`
+agent-evals cache - Manage cached operation entries
+
+Usage:
+  agent-evals cache list [flags]
+  agent-evals cache clear --eval <id>
+  agent-evals cache clear --all
+
+Flags:
+  --eval <id>                Clear entries for specific eval(s) (comma-separated)
+  --all                      Confirm clearing every cached entry
+  --json                     Output cache listing as JSON
+  --help, -h                 Show this help
+  `);
+		return;
+	}
 	console.info(`
 agent-evals - LLM/Agent eval runner
 
@@ -3416,7 +3848,8 @@ Options:
   --no-cache                 Shortcut for --cache bypass
   --refresh-cache            Shortcut for --cache refresh
   --clear-cache              Clear the cache before starting the run
+  --help, -h                 Show help
   `);
 }
 //#endregion
-export { jsonCellSchema as $, scoreTraceSchema as A, traceAttributeDisplayFormatSchema as B, caseDetailSchema as C, evalStatItemSchema as D, evalStatAggregateSchema as E, evalChartConfigSchema as F, traceDisplayInputConfigSchema as G, traceAttributeDisplayPlacementSchema as H, evalChartMetricSchema as I, cellValueSchema as J, traceSpanKindSchema as K, evalChartTooltipExtraSchema as L, evalChartAxisSchema as M, evalChartBuiltinMetricSchema as N, evalStatsConfigSchema as O, evalChartColorSchema as P, fileRefSchema as Q, evalChartTypeSchema as R, assertionFailureSchema as S, evalFreshnessStatusSchema as T, traceAttributeDisplaySchema as U, traceAttributeDisplayInputSchema as V, traceDisplayConfigSchema as W, columnFormatSchema as X, columnDefSchema as Y, columnKindSchema as Z, cacheModeSchema as _, getEvalRegistry as _t, sseEnvelopeSchema as a, evalTracer as at, serializedCacheSpanSchema as b, deriveScopedSummaryFromCases as c, evalAssert as ct, runManifestSchema as d, isInEvalScope as dt, numberDisplayOptionsSchema as et, runSummarySchema as f, runInEvalScope as ft, cacheListItemSchema as g, defineEval as gt, cacheEntrySchema as h, repoFile as ht, updateManualScoreRequestSchema as i, evalSpan as it, evalChartAggregateSchema as j, evalSummarySchema as k, deriveStatusFromCaseRows as l, getCurrentScope as lt, trialSelectionModeSchema as m, setScopeCacheContext as mt, createRunner as n, runArtifactRefSchema as nt, getEvalTitle as o, hashCacheKey as ot, agentEvalsConfigSchema as p, setEvalOutput as pt, traceSpanSchema as q, createRunRequestSchema as r, buildTraceTree as rt, getEvalDisplayStatus as s, EvalAssertionError as st, runCli as t, repoFileRefSchema as tt, deriveStatusFromChildStatuses as u, incrementEvalOutput as ut, cacheRecordingOpSchema as v, caseRowSchema as w, spanCacheOptionsSchema as x, cacheRecordingSchema as y, evalChartsConfigSchema as z };
+export { fileRefSchema as $, evalSummarySchema as A, evalChartsConfigSchema as B, assertionFailureSchema as C, evalStatAggregateSchema as D, evalFreshnessStatusSchema as E, evalChartColorSchema as F, traceDisplayConfigSchema as G, traceAttributeDisplayInputSchema as H, evalChartConfigSchema as I, traceSpanSchema as J, traceDisplayInputConfigSchema as K, evalChartMetricSchema as L, evalChartAggregateSchema as M, evalChartAxisSchema as N, evalStatItemSchema as O, evalChartBuiltinMetricSchema as P, columnKindSchema as Q, evalChartTooltipExtraSchema as R, spanCacheOptionsSchema as S, caseRowSchema as T, traceAttributeDisplayPlacementSchema as U, traceAttributeDisplayFormatSchema as V, traceAttributeDisplaySchema as W, columnDefSchema as X, cellValueSchema as Y, columnFormatSchema as Z, cacheListItemSchema as _, repoFile as _t, sseEnvelopeSchema as a, evalSpan as at, cacheRecordingSchema as b, deriveScopedSummaryFromCases as c, hashCacheKeySync as ct, runManifestSchema as d, getCurrentScope as dt, jsonCellSchema as et, runSummarySchema as f, incrementEvalOutput as ft, cacheFileSchema as g, setScopeCacheContext as gt, cacheEntrySchema as h, setEvalOutput as ht, updateManualScoreRequestSchema as i, buildTraceTree as it, scoreTraceSchema as j, evalStatsConfigSchema as k, deriveStatusFromCaseRows as l, EvalAssertionError as lt, trialSelectionModeSchema as m, runInEvalScope as mt, createRunner as n, repoFileRefSchema as nt, getEvalTitle as o, evalTracer as ot, agentEvalsConfigSchema as p, isInEvalScope as pt, traceSpanKindSchema as q, createRunRequestSchema as r, runArtifactRefSchema as rt, getEvalDisplayStatus as s, hashCacheKey as st, runCli as t, numberDisplayOptionsSchema as tt, deriveStatusFromChildStatuses as u, evalAssert as ut, cacheModeSchema as v, defineEval as vt, caseDetailSchema as w, serializedCacheSpanSchema as x, cacheRecordingOpSchema as y, getEvalRegistry as yt, evalChartTypeSchema as z };