agent-skill-evals 0.1.0 → 0.1.1

package/dist/agent-B15R7Xlr.mjs

@@ -0,0 +1,2105 @@
+import { n as RUNTIME_CHECKS_BY_TYPE } from "./catalog-9Wh-AfRv.mjs";
+import { i as resolveInvocation, n as PRESET_IDS, r as presetStalenessHint } from "./presets-DwfDWckA.mjs";
+import { z } from "zod";
+import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
+import { chmod, copyFile, cp, mkdir, mkdtemp, readFile, readdir, stat, writeFile } from "node:fs/promises";
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { createHash } from "node:crypto";
+import { createServer } from "node:net";
+//#region src/assertion-entries.ts
+function isEntryObject(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
+* Normalizes generated Promptfoo `vars.preconditions | expect` entries.
+*
+* Supports the clean Test Pack shorthand form:
+* `{ "file.exists": { path: "app.js" } }`
+*/
+function parseRuntimeTestFields(vars) {
+	const preconditions = parseAssertionEntries(vars.preconditions, "preconditions", { allowMissing: true });
+	const expect = parseAssertionEntries(vars.expect, "expect", { allowMissing: true });
+	return {
+		preconditions: preconditions.entries,
+		expect: expect.entries,
+		errors: [...preconditions.errors, ...expect.errors]
+	};
+}
+function parseAssertionEntries(raw, field, options = {}) {
+	const entries = [];
+	const errors = [];
+	if (raw === void 0 || raw === null) {
+		if (!options.allowMissing) errors.push({
+			field,
+			reason: "must be an array of assertion entries"
+		});
+		return {
+			entries,
+			errors
+		};
+	}
+	if (!Array.isArray(raw)) return {
+		entries,
+		errors: [{
+			field,
+			reason: "must be an array of assertion entries"
+		}]
+	};
+	raw.forEach((entry, index) => {
+		const parsed = parseAssertionEntry(entry, field, index);
+		if ("error" in parsed) errors.push(parsed.error);
+		else entries.push(parsed.entry);
+	});
+	return {
+		entries,
+		errors
+	};
+}
+function parseAssertionEntry(entry, field, index) {
+	if (!isEntryObject(entry)) return { error: {
+		field,
+		index,
+		reason: "entry must be a shorthand assertion object"
+	} };
+	const keys = Object.keys(entry);
+	if (keys.length !== 1) return { error: {
+		field,
+		index,
+		reason: "shorthand assertion object must have exactly one key"
+	} };
+	const type = keys[0];
+	const args = entry[type];
+	if (!isEntryObject(args)) return { error: {
+		field,
+		index,
+		reason: `shorthand assertion "${type}" value must be an object`
+	} };
+	return { entry: {
+		type,
+		args
+	} };
+}
+//#endregion
+//#region src/evidence-schema.ts
+const EVIDENCE_SCHEMA_VERSION = "agent-skill-evals.evidence.v2";
+const OptionalString = z.string().optional();
+const OptionalNumber = z.number().optional();
+const StringArray = z.array(z.string());
+const CommandEventSchema = z.object({
+	command: z.string(),
+	args: z.array(z.string()).default([]),
+	exitCode: z.number(),
+	signal: OptionalString,
+	stdout: OptionalString,
+	stderr: OptionalString,
+	startedAt: z.number(),
+	durationMs: z.number(),
+	turn: OptionalNumber
+});
+const FileEventSchema = z.object({
+	path: z.string(),
+	op: z.enum([
+		"create",
+		"modify",
+		"delete"
+	])
+});
+const ToolCallEventSchema = z.object({
+	tool: z.string(),
+	provider: OptionalString,
+	server: OptionalString,
+	args: z.unknown().optional(),
+	result: z.unknown().optional(),
+	startedAt: z.number(),
+	durationMs: z.number(),
+	turn: OptionalNumber
+});
+const SkillLoadEventSchema = z.object({
+	skill: z.string(),
+	delivery: z.enum([
+		"native",
+		"mcp",
+		"explicit"
+	]),
+	provider: OptionalString,
+	server: OptionalString,
+	source: OptionalString,
+	startedAt: z.number()
+});
+const SkillAvailableEventSchema = z.object({
+	skill: z.string(),
+	path: z.string(),
+	role: z.enum([
+		"under-test",
+		"supporting",
+		"distractor"
+	])
+});
+const UsageSchema = z.object({
+	inputTokens: OptionalNumber,
+	outputTokens: OptionalNumber,
+	totalTokens: OptionalNumber,
+	cacheReadTokens: OptionalNumber,
+	cacheWriteTokens: OptionalNumber
+});
+const RunSummarySchema = z.object({
+	runDir: z.string(),
+	worldPath: z.string(),
+	fixture: z.string().optional(),
+	durationMs: OptionalNumber
+});
+const RuntimeIdentitySchema = z.object({
+	adapter: OptionalString,
+	preset: OptionalString,
+	command: OptionalString,
+	cliVersion: OptionalString,
+	model: OptionalString,
+	continuation: z.enum(["transcript-replay", "native-session"]).optional()
+});
+const TurnRecordSchema = z.object({
+	turn: z.number(),
+	role: z.enum(["user", "agent"]),
+	text: z.string(),
+	startedAt: z.number(),
+	durationMs: z.number(),
+	usage: UsageSchema.optional()
+});
+const EvidenceSnapshotSchema = z.object({
+	schemaVersion: z.literal(EVIDENCE_SCHEMA_VERSION),
+	output: z.string().default(""),
+	run: RunSummarySchema,
+	commands: z.array(CommandEventSchema).default([]),
+	filesWritten: z.array(FileEventSchema).default([]),
+	toolCalls: z.array(ToolCallEventSchema).default([]),
+	skillsLoaded: z.array(SkillLoadEventSchema).default([]),
+	skillsAvailable: z.array(SkillAvailableEventSchema).default([]),
+	usage: UsageSchema.default({}),
+	turns: z.array(TurnRecordSchema).optional(),
+	runtime: RuntimeIdentitySchema.optional(),
+	warnings: StringArray.optional(),
+	extensions: z.record(z.string(), z.unknown()).optional()
+});
+function decodeEvidenceSnapshot(input) {
+	const decoded = EvidenceSnapshotSchema.safeParse(input);
+	if (decoded.success) return {
+		ok: true,
+		value: decoded.data
+	};
+	return {
+		ok: false,
+		error: /* @__PURE__ */ new Error(`the evidence file does not match the agent-skill-evals.evidence.v2 shape. It was probably written by a different agent-skill-evals version or edited by hand — re-run the eval to regenerate it. Details: ${z.prettifyError(decoded.error)}`)
+	};
+}
+function parseEvidenceSnapshot(input) {
+	const decoded = decodeEvidenceSnapshot(input);
+	if (decoded.ok) return decoded.value;
+	throw decoded.error;
+}
+//#endregion
+//#region src/agent/evidence.ts
+const DEFAULT_SKILL_EVIDENCE_CONFIG = {
+	mcpResource: {
+		uriArgPaths: ["uri"],
+		uriPatterns: ["^skill://(?<skill>[^/]+)/SKILL\\.md$"]
+	},
+	mcpTool: { toolPatterns: ["^load_(?<skill>[A-Za-z0-9_-]+)_skill$"] }
+};
+var EvidenceCollector = class EvidenceCollector {
+	skillEvidenceConfig;
+	currentTurn;
+	/** Usage from completed turns; the live snapshot.usage covers only the current turn. */
+	completedTurnsUsage = {};
+	snapshot = {
+		schemaVersion: EVIDENCE_SCHEMA_VERSION,
+		output: "",
+		run: {
+			runDir: "",
+			worldPath: "",
+			fixture: ""
+		},
+		commands: [],
+		filesWritten: [],
+		toolCalls: [],
+		skillsLoaded: [],
+		skillsAvailable: [],
+		usage: {}
+	};
+	constructor(skillEvidenceConfig = {}) {
+		this.skillEvidenceConfig = mergeSkillEvidenceConfig(skillEvidenceConfig);
+	}
+	addCommand(e) {
+		this.snapshot.commands.push(this.withTurn(e));
+	}
+	addFileWrite(e) {
+		this.snapshot.filesWritten.push(e);
+	}
+	addToolCall(e) {
+		const event = this.withTurn(e);
+		this.snapshot.toolCalls.push(event);
+		const skillLoad = skillLoadFromToolCall(event, this.skillEvidenceConfig);
+		if (skillLoad) this.addSkillLoad(skillLoad);
+	}
+	withTurn(event) {
+		return this.currentTurn === void 0 ? event : {
+			...event,
+			turn: this.currentTurn
+		};
+	}
+	/**
+	* Start an agent turn in a multi-turn run. Adapters overwrite usage per
+	* CLI invocation (setUsage), so each turn's usage is scoped: the previous
+	* turn's counts move into the completed-turns total first.
+	*/
+	beginAgentTurn(turn) {
+		this.completedTurnsUsage = mergeUsage(this.completedTurnsUsage, this.snapshot.usage);
+		this.snapshot.usage = {};
+		this.currentTurn = turn;
+	}
+	endAgentTurn(record) {
+		if (this.currentTurn === void 0) return;
+		this.addTurnRecord({
+			turn: this.currentTurn,
+			role: "agent",
+			text: record.text,
+			startedAt: record.startedAt,
+			durationMs: record.durationMs,
+			usage: { ...this.snapshot.usage }
+		});
+	}
+	addUserTurn(turn, text, startedAt) {
+		this.addTurnRecord({
+			turn,
+			role: "user",
+			text,
+			startedAt,
+			durationMs: 0
+		});
+	}
+	addTurnRecord(record) {
+		const turns = this.snapshot.turns ?? [];
+		turns.push(record);
+		this.snapshot.turns = turns;
+	}
+	addSkillLoad(e) {
+		if (this.snapshot.skillsLoaded.some((existing) => existing.skill === e.skill && existing.delivery === e.delivery && existing.provider === e.provider && existing.server === e.server)) return;
+		this.snapshot.skillsLoaded.push(e);
+	}
+	addSkillAvailable(e) {
+		if (!this.snapshot.skillsAvailable.some((existing) => existing.skill === e.skill && existing.path === e.path)) this.snapshot.skillsAvailable.push(e);
+	}
+	setUsage(u) {
+		this.snapshot.usage = u;
+	}
+	/** Merge non-empty runtime identity fields; later observations win. */
+	mergeRuntime(identity) {
+		const merged = { ...this.snapshot.runtime };
+		for (const [key, value] of Object.entries(identity)) if (typeof value === "string" && value.length > 0) merged[key] = value;
+		if (Object.keys(merged).length > 0) this.snapshot.runtime = merged;
+	}
+	addWarning(warning) {
+		const warnings = this.snapshot.warnings ?? [];
+		if (warnings.includes(warning)) return;
+		warnings.push(warning);
+		this.snapshot.warnings = warnings;
+	}
+	addUsage(u) {
+		this.snapshot.usage = mergeUsage(this.snapshot.usage, u);
+	}
+	setOutput(output) {
+		this.snapshot.output = output;
+	}
+	setRun(run) {
+		this.snapshot.run = run;
+	}
+	toSnapshot() {
+		return parseEvidenceSnapshot({
+			schemaVersion: this.snapshot.schemaVersion,
+			output: this.snapshot.output,
+			run: { ...this.snapshot.run },
+			commands: [...this.snapshot.commands],
+			filesWritten: [...this.snapshot.filesWritten],
+			toolCalls: [...this.snapshot.toolCalls],
+			skillsLoaded: [...this.snapshot.skillsLoaded],
+			skillsAvailable: [...this.snapshot.skillsAvailable],
+			usage: mergeUsage(this.completedTurnsUsage, this.snapshot.usage),
+			turns: this.snapshot.turns ? this.snapshot.turns.map((t) => ({ ...t })) : void 0,
+			runtime: this.snapshot.runtime ? { ...this.snapshot.runtime } : void 0,
+			warnings: this.snapshot.warnings ? [...this.snapshot.warnings] : void 0,
+			extensions: this.snapshot.extensions ? { ...this.snapshot.extensions } : void 0
+		});
+	}
+	static fromSnapshot(snapshot) {
+		const collector = new EvidenceCollector();
+		collector.snapshot = parseEvidenceSnapshot(snapshot);
+		return collector;
+	}
+};
+function mergeSkillEvidenceConfig(config) {
+	return {
+		mcpResource: {
+			uriArgPaths: config.mcpResource?.uriArgPaths ?? DEFAULT_SKILL_EVIDENCE_CONFIG.mcpResource.uriArgPaths,
+			uriPatterns: config.mcpResource?.uriPatterns ?? DEFAULT_SKILL_EVIDENCE_CONFIG.mcpResource.uriPatterns
+		},
+		mcpTool: { toolPatterns: config.mcpTool?.toolPatterns ?? DEFAULT_SKILL_EVIDENCE_CONFIG.mcpTool.toolPatterns },
+		...config.nativeArgs ? { nativeArgs: config.nativeArgs } : {}
+	};
+}
+function skillLoadFromToolCall(event, config) {
+	const uri = skillUriFromArgs(event.args, config.mcpResource?.uriArgPaths ?? []);
+	const skill = uri ? skillFromPattern(uri, config.mcpResource?.uriPatterns ?? []) : skillFromTool(event.tool, config.mcpTool?.toolPatterns ?? []);
+	if (!skill) return void 0;
+	const server = event.server ?? serverFromArgs(event.args);
+	return {
+		skill,
+		delivery: "mcp",
+		...event.provider ? { provider: event.provider } : {},
+		...server ? { server } : {},
+		source: event.tool,
+		startedAt: event.startedAt
+	};
+}
+function skillUriFromArgs(args, paths) {
+	for (const path of paths) {
+		const uri = valueAtPath(args, path);
+		if (typeof uri === "string") return uri;
+	}
+}
+function serverFromArgs(args) {
+	if (!args || typeof args !== "object" || Array.isArray(args)) return void 0;
+	const server = args.server;
+	return typeof server === "string" ? server : void 0;
+}
+function skillFromPattern(value, patterns) {
+	for (const pattern of patterns) {
+		let regex;
+		try {
+			regex = new RegExp(pattern, "i");
+		} catch {
+			continue;
+		}
+		const match = regex.exec(value);
+		const skill = match?.groups?.skill ?? match?.[1];
+		if (skill) return skill;
+	}
+}
+function skillFromTool(tool, patterns) {
+	return skillFromPattern(tool, patterns)?.replace(/_/g, "-");
+}
+function valueAtPath(value, path) {
+	let current = value;
+	for (const segment of path.split(".")) {
+		if (!current || typeof current !== "object" || Array.isArray(current)) return void 0;
+		current = current[segment];
+	}
+	return current;
+}
+function addOptionalNumbers(a, b) {
+	if (a === void 0) return b;
+	if (b === void 0) return a;
+	return a + b;
+}
+function mergeUsage(a, b) {
+	return {
+		inputTokens: addOptionalNumbers(a.inputTokens, b.inputTokens),
+		outputTokens: addOptionalNumbers(a.outputTokens, b.outputTokens),
+		totalTokens: addOptionalNumbers(a.totalTokens, b.totalTokens),
+		cacheReadTokens: addOptionalNumbers(a.cacheReadTokens, b.cacheReadTokens),
+		cacheWriteTokens: addOptionalNumbers(a.cacheWriteTokens, b.cacheWriteTokens)
+	};
+}
+async function writeEvidenceTo(collector, runDir) {
+	const path = join(runDir, "evidence.json");
+	await writeFile(path, JSON.stringify(collector.toSnapshot(), null, 2));
+	return path;
+}
+function evidenceFromSnapshot(s) {
+	return {
+		output: () => s.output,
+		commands: () => s.commands,
+		filesWritten: () => s.filesWritten,
+		toolCalls: () => s.toolCalls,
+		skillsLoaded: () => s.skillsLoaded,
+		skillsAvailable: () => s.skillsAvailable,
+		usage: () => s.usage,
+		turns: () => s.turns ?? [],
+		runtime: () => s.runtime ?? {},
+		warnings: () => s.warnings ?? []
+	};
+}
+//#endregion
+//#region src/agent/command-runner.ts
+const DEFAULT_OUTPUT_LIMIT = 4096;
+const KILL_GRACE_MS = 1e3;
+const EXIT_STDIO_FLUSH_MS = 50;
+function appendLimited(current, chunk, limit) {
+	if (limit <= 0) return "";
+	const next = current + chunk;
+	return next.length <= limit ? next : next.slice(next.length - limit);
+}
+function killProcessGroup(child, signal) {
+	if (!child.pid) return;
+	try {
+		if (process.platform === "win32") child.kill(signal);
+		else process.kill(-child.pid, signal);
+	} catch {
+		try {
+			child.kill(signal);
+		} catch {}
+	}
+}
+function runCommand(command, args = [], options) {
+	const stdoutLimit = options.stdoutLimit ?? DEFAULT_OUTPUT_LIMIT;
+	const stderrLimit = options.stderrLimit ?? DEFAULT_OUTPUT_LIMIT;
+	const startedAt = Date.now();
+	return new Promise((resolve) => {
+		const child = spawn(command, [...args], {
+			cwd: options.cwd,
+			env: options.env,
+			detached: process.platform !== "win32",
+			stdio: [
+				"pipe",
+				"pipe",
+				"pipe"
+			]
+		});
+		let stdout = "";
+		let stderr = "";
+		let timedOut = false;
+		let settled = false;
+		let timeoutTimer = null;
+		let forceTimer = null;
+		let exitFlushTimer = null;
+		const finish = (result) => {
+			if (settled) return;
+			settled = true;
+			if (timeoutTimer) clearTimeout(timeoutTimer);
+			if (forceTimer) clearTimeout(forceTimer);
+			if (exitFlushTimer) clearTimeout(exitFlushTimer);
+			child.stdout?.destroy();
+			child.stderr?.destroy();
+			child.stdin?.destroy();
+			if (process.platform !== "win32") killProcessGroup(child, "SIGTERM");
+			resolve({
+				...result,
+				durationMs: Date.now() - startedAt
+			});
+		};
+		child.stdout?.on("data", (chunk) => {
+			const text = chunk.toString();
+			stdout = appendLimited(stdout, text, stdoutLimit);
+			options.onStdout?.(text);
+		});
+		child.stderr?.on("data", (chunk) => {
+			const text = chunk.toString();
+			stderr = appendLimited(stderr, text, stderrLimit);
+			options.onStderr?.(text);
+		});
+		child.on("error", (error) => {
+			stderr = appendLimited(stderr, String(error), stderrLimit);
+			finish({
+				command,
+				args: [...args],
+				exitCode: -1,
+				stdout,
+				stderr,
+				startedAt,
+				timedOut,
+				error
+			});
+		});
+		child.on("exit", (code, signal) => {
+			const result = {
+				command,
+				args: [...args],
+				exitCode: code ?? -1,
+				signal: signal ?? void 0,
+				stdout,
+				stderr,
+				startedAt,
+				timedOut
+			};
+			exitFlushTimer = setTimeout(() => finish(result), EXIT_STDIO_FLUSH_MS);
+		});
+		child.on("close", (code, signal) => {
+			finish({
+				command,
+				args: [...args],
+				exitCode: code ?? -1,
+				signal: signal ?? void 0,
+				stdout,
+				stderr,
+				startedAt,
+				timedOut
+			});
+		});
+		if (options.stdin !== void 0) child.stdin?.write(options.stdin);
+		child.stdin?.end();
+		if (options.timeoutMs) timeoutTimer = setTimeout(() => {
+			timedOut = true;
+			stderr = appendLimited(stderr, `${stderr ? "\n" : ""}agent-skill-evals: command timed out after ${options.timeoutMs}ms`, stderrLimit);
+			killProcessGroup(child, "SIGKILL");
+			forceTimer = setTimeout(() => {
+				finish({
+					command,
+					args: [...args],
+					exitCode: -1,
+					signal: "SIGKILL",
+					stdout,
+					stderr,
+					startedAt,
+					timedOut
+				});
+			}, KILL_GRACE_MS);
+		}, options.timeoutMs);
+	});
+}
+//#endregion
+//#region src/agent/jsonl-stream.ts
+function appendLine(events, line) {
+	if (!line.trim()) return events;
+	try {
+		return [...events, JSON.parse(line)];
+	} catch {
+		return events;
+	}
+}
+function parseChunk(state, text) {
+	const lines = (state.leftover + text).split("\n");
+	return {
+		leftover: lines.pop() ?? "",
+		events: lines.reduce(appendLine, state.events)
+	};
+}
+function createJsonlEventParser() {
+	let leftover = "";
+	return {
+		push(chunk) {
+			const state = parseChunk({
+				leftover,
+				events: []
+			}, chunk);
+			leftover = state.leftover;
+			return state.events;
+		},
+		finish() {
+			const events = appendLine([], leftover);
+			leftover = "";
+			return events;
+		}
+	};
+}
+//#endregion
+//#region src/agent/adapters.ts
+const pendingPiToolCalls = /* @__PURE__ */ new WeakMap();
+function argsWithPrompt(args, prompt) {
+	const dashIndex = args.lastIndexOf("-");
+	if (dashIndex === -1) return [...args, prompt];
+	return args.map((arg, index) => index === dashIndex ? prompt : arg);
+}
+function normalizePathFromCwd(path, cwd) {
+	if (!isAbsolute(path)) return path;
+	const candidate = path.startsWith("/private/") ? path.slice(8) : path;
+	const rel = relative(cwd.startsWith("/private/") ? cwd.slice(8) : cwd, candidate);
+	return rel && !rel.startsWith("..") && !isAbsolute(rel) ? rel : path;
+}
+function normalizeToolCallArgs(args, cwd) {
+	if (!args || typeof args !== "object" || Array.isArray(args)) return args;
+	const normalized = {};
+	for (const [key, value] of Object.entries(args)) {
+		const normalizedValue = (key === "path" || key === "file_path") && typeof value === "string" ? normalizePathFromCwd(value, cwd) : value;
+		normalized[key] = normalizedValue;
+		if (key === "file_path" && typeof normalizedValue === "string" && normalized.path === void 0) normalized.path = normalizedValue;
+	}
+	return normalized;
+}
+function normalizeMcpToolCall(event) {
+	const match = /^mcp__(.+?)__(.+)$/.exec(event.tool);
+	if (!match) return event;
+	return {
+		...event,
+		server: event.server ?? match[1],
+		tool: match[2] ?? event.tool
+	};
+}
+function parseJsonObjectString(value) {
+	if (typeof value !== "string") return value;
+	try {
+		const parsed = JSON.parse(value);
+		return parsed && typeof parsed === "object" ? parsed : value;
+	} catch {
+		return value;
+	}
+}
+function evidenceWithRelativeToolPaths(evidence, cwd, now) {
+	return {
+		addCommand: (event) => evidence.addCommand(event),
+		addToolCall: (event) => {
+			const normalized = normalizeMcpToolCall(event);
+			evidence.addToolCall({
+				...normalized,
+				args: normalizeToolCallArgs(normalized.args, cwd)
+			});
+		},
+		setUsage: (usage) => evidence.setUsage(usage),
+		addUsage: (usage) => evidence.addUsage(usage),
+		setModel: (model) => evidence.mergeRuntime({ model }),
+		now
+	};
+}
+function normalizeUsage(usage) {
+	const inputTokens = usage.input_tokens ?? usage.inputTokens ?? usage.input;
+	const outputTokens = usage.output_tokens ?? usage.outputTokens ?? usage.output;
+	const cacheReadTokens = usage.cache_read_input_tokens ?? usage.cacheReadTokens ?? usage.cacheRead;
+	const cacheWriteTokens = usage.cache_creation_input_tokens ?? usage.cacheWriteTokens ?? usage.cacheWrite;
+	return {
+		inputTokens,
+		outputTokens,
+		totalTokens: usage.total_tokens ?? usage.totalTokens ?? addKnownNumbers(inputTokens, outputTokens),
+		cacheReadTokens,
+		cacheWriteTokens
+	};
+}
+function addKnownNumbers(...values) {
+	const present = values.filter((value) => typeof value === "number");
+	return present.length > 0 ? present.reduce((sum, value) => sum + value, 0) : void 0;
+}
+function normalizePiToolName(tool) {
+	switch (tool.toLowerCase()) {
+		case "bash": return "Bash";
+		case "edit": return "Edit";
+		case "read": return "Read";
+		case "write": return "Write";
+		default: return tool;
+	}
+}
+function evidenceStartedAt(evidence) {
+	return evidence.now?.() ?? 0;
+}
+async function runJsonlAdapter(input, onEvent, options = {}) {
+	const { command, args, cwd, prompt, evidence, timeoutMs, env } = input;
+	const promptDelivery = options.promptDelivery ?? "stdin";
+	const spawnArgs = promptDelivery === "arg" ? argsWithPrompt(args, prompt) : [...args];
+	const runStartedAt = Date.now();
+	const adapterEvidence = evidenceWithRelativeToolPaths(evidence, cwd, () => runStartedAt);
+	const parser = createJsonlEventParser();
+	const unrecognized = /* @__PURE__ */ new Map();
+	let finalText = "";
+	let terminalError;
+	const dispatch = (evt) => {
+		terminalError = options.terminalError?.(evt) ?? terminalError;
+		if (onEvent(evt, adapterEvidence, (text, replace = false) => {
+			finalText = replace ? text : finalText + text;
+		})) return;
+		const type = evt && typeof evt === "object" && typeof evt.type === "string" ? evt.type : "<no type>";
+		unrecognized.set(type, (unrecognized.get(type) ?? 0) + 1);
+	};
+	const handleChunk = (chunk) => {
+		for (const evt of parser.push(chunk)) dispatch(evt);
+	};
+	const result = await runCommand(command, spawnArgs, {
+		cwd,
+		env,
+		stdin: promptDelivery === "stdin" ? prompt : void 0,
+		timeoutMs,
+		stdoutLimit: 0,
+		onStdout: handleChunk
+	});
+	for (const evt of parser.finish()) dispatch(evt);
+	if (unrecognized.size > 0) {
+		const summary = [...unrecognized.entries()].map(([type, count]) => `${type} (${count})`).join(", ");
+		evidence.addWarning(`${options.adapterId ?? "adapter"}: unrecognized event type(s) in CLI output: ${summary}. The CLI's JSON schema may have changed since this adapter was written; recorded evidence may be incomplete.`);
+	}
+	evidence.addCommand({
+		command,
+		args: [...spawnArgs],
+		exitCode: result.exitCode,
+		stderr: result.stderr.slice(0, 4096),
+		startedAt: result.startedAt,
+		durationMs: result.durationMs
+	});
+	const error = result.error ? `adapter error: failed to start "${command}": ${result.error.message}` : result.timedOut ? `${command} timed out after ${timeoutMs ?? 0}ms` : result.exitCode !== 0 ? `${command} exited ${result.exitCode}${result.stderr.trim() ? `: ${result.stderr.trim()}` : ""}` : terminalError;
+	if (error && !finalText.trim()) finalText = error;
+	return {
+		output: finalText.trim(),
+		exitCode: result.exitCode,
+		timedOut: result.timedOut,
+		...error ? { error } : {},
+		durationMs: result.durationMs
+	};
+}
+/**
+* Claude Code stream-json adapter: parses stream-json events emitted by
+* `claude -p ... --output-format stream-json` and projects them into evidence.
+*
+* Events of interest (see Claude Code docs):
+*  - { type: "system" | "assistant" | "user" | "result", ... }
+*  - tool_use blocks inside assistant content (Bash, Edit, Write, etc.)
+*/
+const claudeCodeJsonAdapter = {
+	id: "claude-code-json",
+	async run(input) {
+		const result = await runJsonlAdapter(input, handleClaudeEvent, { adapterId: "claude-code-json" });
+		if (result.exitCode !== 0 && /not logged in/i.test(result.output)) {
+			const hint = "Claude Code runs with an isolated HOME and cannot reuse a macOS Keychain login. Set CLAUDE_CODE_OAUTH_TOKEN (created by `claude setup-token`) or ANTHROPIC_API_KEY.";
+			return {
+				...result,
+				output: `${result.output}\n\n${hint}`,
+				error: `${result.error ?? "Claude Code authentication failed"}. ${hint}`
+			};
+		}
+		return result;
+	}
+};
+const codexJsonAdapter = {
+	id: "codex-json",
+	run(input) {
+		return runJsonlAdapter(input, handleCodexEvent, {
+			promptDelivery: "arg",
+			adapterId: "codex-json"
+		});
+	}
+};
+const piJsonAdapter = {
+	id: "pi-json",
+	async run(input) {
+		const result = await runJsonlAdapter(input, handlePiEvent, {
+			adapterId: "pi-json",
+			terminalError: piTerminalError
+		});
+		if (result.error && /no api key for provider/i.test(result.error)) return {
+			...result,
+			error: `Pi authentication failed: ${result.error}`
+		};
+		return result;
+	}
+};
+function piTerminalError(event) {
+	if (!event || typeof event !== "object") return void 0;
+	const message = event.message;
+	if (!message || typeof message !== "object") return void 0;
+	const terminal = message;
+	return terminal.stopReason === "error" && typeof terminal.errorMessage === "string" ? terminal.errorMessage : void 0;
+}
+/** Event types Claude Code's stream-json output is known to emit. */
+const CLAUDE_KNOWN_EVENT_TYPES = new Set([
+	"system",
+	"assistant",
+	"user",
+	"result",
+	"stream_event",
+	"rate_limit_event"
+]);
+function handleClaudeEvent(evt, evidence, appendFinal) {
+	if (!evt || typeof evt !== "object") return false;
+	const e = evt;
+	if (e.type === "system" && typeof e.model === "string") evidence.setModel?.(e.model);
+	if (e.type === "result" && typeof e.result === "string") appendFinal(e.result);
+	if (e.type === "assistant" && e.message && typeof e.message === "object") {
+		const msg = e.message;
+		if (typeof msg.model === "string") evidence.setModel?.(msg.model);
+		if (msg.usage && typeof msg.usage === "object") evidence.addUsage(normalizeUsage(msg.usage));
+		if (Array.isArray(msg.content)) for (const block of msg.content) {
+			if (!block || typeof block !== "object") continue;
+			const b = block;
+			if (b.type === "tool_use" && typeof b.name === "string") evidence.addToolCall({
+				tool: b.name,
+				provider: "claude-code-json",
+				args: b.input,
+				startedAt: evidenceStartedAt(evidence),
+				durationMs: 0
+			});
+		}
+	}
+	if (e.type === "result" && e.usage && typeof e.usage === "object") evidence.setUsage(normalizeUsage(e.usage));
+	return typeof e.type === "string" && CLAUDE_KNOWN_EVENT_TYPES.has(e.type);
+}
+/**
+* Codex lifecycle event families that carry no evidence but are normal
+* output — recognized so they do not trigger schema-drift warnings.
+*/
+const CODEX_KNOWN_TYPE_PATTERN = /^(thread|turn|session|item|task|agent|token_count|notification|status|reasoning|error|usage)([._:]|$)/;
+function handleCodexEvent(evt, evidence, appendFinal) {
+	if (!evt || typeof evt !== "object") return false;
+	const e = evt;
+	const type = typeof e.type === "string" ? e.type : "";
+	let consumed = false;
+	if (typeof e.model === "string") evidence.setModel?.(e.model);
+	const text = typeof e.message === "string" ? e.message : typeof e.text === "string" ? e.text : typeof e.content === "string" ? e.content : typeof e.output === "string" ? e.output : void 0;
+	if (text && /message|final|result|response|output/.test(type)) {
+		appendFinal(text);
+		consumed = true;
+	}
+	const item = e.item && typeof e.item === "object" ? e.item : e;
+	const itemType = typeof item.type === "string" ? item.type : "";
+	const toolName = typeof item.tool === "string" ? item.tool : typeof item.name === "string" && /tool|call|command/.test(`${type}:${itemType}`) ? item.name : void 0;
+	if (toolName && /tool|call|command/.test(`${type}:${itemType}`)) {
+		evidence.addToolCall({
+			tool: toolName,
+			provider: "codex-json",
+			server: typeof item.server === "string" ? item.server : void 0,
+			args: parseJsonObjectString(item.args ?? item.input ?? item.arguments),
+			result: item.result ?? item.output,
+			startedAt: evidenceStartedAt(evidence),
+			durationMs: 0
+		});
+		consumed = true;
+	}
+	if ((type === "file_change" || type === "event" || /(?:completed|failed|result)$/.test(type)) && itemType === "file_change" && Array.isArray(item.changes)) {
+		for (const change of item.changes) {
+			if (!change || typeof change !== "object") continue;
+			const c = change;
+			if (typeof c.path !== "string") continue;
+			evidence.addToolCall({
+				tool: "Edit",
+				provider: "codex-json",
+				args: {
+					path: c.path,
+					kind: typeof c.kind === "string" ? c.kind : void 0
+				},
+				startedAt: evidenceStartedAt(evidence),
+				durationMs: 0
+			});
+		}
+		consumed = true;
+	}
+	if ((type === "exec_command" || /(?:completed|failed|result)$/.test(type)) && (type.includes("exec") || type.includes("command") || itemType.includes("command")) && typeof item.command === "string") {
+		const args = Array.isArray(item.args) ? item.args.map(String) : [];
+		evidence.addCommand({
+			command: item.command,
+			args,
+			exitCode: typeof item.exit_code === "number" ? item.exit_code : typeof item.exitCode === "number" ? item.exitCode : 0,
+			stdout: typeof item.stdout === "string" ? item.stdout.slice(0, 4096) : typeof item.aggregated_output === "string" ? item.aggregated_output.slice(0, 4096) : void 0,
+			stderr: typeof item.stderr === "string" ? item.stderr.slice(0, 4096) : void 0,
+			startedAt: evidenceStartedAt(evidence),
+			durationMs: typeof item.durationMs === "number" ? item.durationMs : 0
+		});
+		consumed = true;
+	}
+	const usage = e.usage && typeof e.usage === "object" ? e.usage : void 0;
+	if (usage) {
+		evidence.setUsage(normalizeUsage(usage));
+		consumed = true;
+	}
+	return consumed || CODEX_KNOWN_TYPE_PATTERN.test(type);
+}
+/** Pi lifecycle events that carry no evidence but are normal output. */
+const PI_KNOWN_EVENT_TYPES = new Set([
+	"message_start",
+	"message_update",
+	"message_end",
+	"turn_start",
+	"turn_end",
+	"agent_start",
+	"agent_end",
+	"tool_execution_start",
+	"tool_execution_update",
+	"tool_execution_end",
+	"session",
+	"thinking",
+	"usage",
+	"final",
+	"error"
+]);
+function handlePiEvent(evt, evidence, appendFinal) {
+	if (!evt || typeof evt !== "object") return false;
+	const e = evt;
+	const type = typeof e.type === "string" ? e.type : "";
+	let consumed = false;
+	const message = e.message && typeof e.message === "object" ? e.message : void 0;
+	if (typeof message?.model === "string") evidence.setModel?.(message.model);
+	const messageUsage = message?.usage && typeof message.usage === "object" ? message.usage : void 0;
+	if (messageUsage && (type === "message_end" || type === "turn_end" || type === "agent_end")) {
+		evidence.setUsage(normalizeUsage(messageUsage));
+		consumed = true;
+	}
+	if (type === "message_end" && message?.role === "assistant" && Array.isArray(message.content)) {
+		let assistantText = "";
+		for (const block of message.content) {
+			if (!block || typeof block !== "object") continue;
+			const content = block;
+			if (content.type === "text" && typeof content.text === "string") assistantText += content.text;
+		}
+		if (assistantText) {
+			appendFinal(assistantText, true);
+			consumed = true;
+		}
+		if (message.stopReason === "error" && typeof message.errorMessage === "string") {
+			appendFinal(message.errorMessage, true);
+			consumed = true;
+		}
+	}
+	const text = typeof e.message === "string" ? e.message : typeof e.text === "string" ? e.text : typeof e.content === "string" ? e.content : typeof e.output === "string" ? e.output : typeof e.result === "string" ? e.result : void 0;
+	if (text && /assistant|message|final|result|response|output/.test(type)) {
+		appendFinal(text);
+		consumed = true;
+	}
+	if (type === "tool_execution_start" || type === "tool_execution_end") {
+		const rawTool = typeof e.tool === "string" ? e.tool : typeof e.name === "string" ? e.name : typeof e.tool_name === "string" ? e.tool_name : typeof e.toolName === "string" ? e.toolName : void 0;
+		const tool = rawTool ? normalizePiToolName(rawTool) : void 0;
+		if (tool && type === "tool_execution_start") {
+			const pending = pendingPiToolCalls.get(evidence) ?? [];
+			pending.push({
+				tool,
+				provider: "pi-json",
+				args: e.args ?? e.input ?? e.arguments,
+				startedAt: evidenceStartedAt(evidence),
+				durationMs: 0
+			});
+			pendingPiToolCalls.set(evidence, pending);
+			consumed = true;
+		}
+		if (tool && type === "tool_execution_end") {
+			const pending = pendingPiToolCalls.get(evidence) ?? [];
+			const index = pending.map((call) => call.tool).lastIndexOf(tool);
+			const started = index >= 0 ? pending.splice(index, 1)[0] : void 0;
+			if (pending.length === 0) pendingPiToolCalls.delete(evidence);
+			evidence.addToolCall({
+				tool,
+				provider: "pi-json",
+				args: e.args ?? e.input ?? e.arguments ?? started?.args,
+				result: e.result ?? e.output,
+				startedAt: started?.startedAt ?? evidenceStartedAt(evidence),
+				durationMs: typeof e.duration_ms === "number" ? e.duration_ms : typeof e.durationMs === "number" ? e.durationMs : 0
+			});
+			consumed = true;
+		}
+	}
+	if (type === "tool_execution_end") {
+		const tool = typeof e.tool === "string" ? e.tool : typeof e.name === "string" ? e.name : typeof e.tool_name === "string" ? e.tool_name : typeof e.toolName === "string" ? e.toolName : "";
+		const commandText = typeof e.command === "string" ? e.command : e.args && typeof e.args === "object" && !Array.isArray(e.args) && typeof e.args.command === "string" ? e.args.command : void 0;
+		if (commandText && /bash|shell|command|exec/i.test(tool)) evidence.addCommand({
+			command: commandText,
+			args: [],
+			exitCode: typeof e.exit_code === "number" ? e.exit_code : typeof e.exitCode === "number" ? e.exitCode : 0,
+			stdout: typeof e.stdout === "string" ? e.stdout.slice(0, 4096) : typeof e.output === "string" ? e.output.slice(0, 4096) : void 0,
+			stderr: typeof e.stderr === "string" ? e.stderr.slice(0, 4096) : void 0,
+			startedAt: evidenceStartedAt(evidence),
+			durationMs: typeof e.duration_ms === "number" ? e.duration_ms : typeof e.durationMs === "number" ? e.durationMs : 0
+		});
+	}
+	const usage = e.usage && typeof e.usage === "object" ? e.usage : e;
+	if (type === "usage" || e.usage) {
+		evidence.setUsage(normalizeUsage(usage));
+		consumed = true;
+	}
+	return consumed || PI_KNOWN_EVENT_TYPES.has(type);
+}
+const adapterRegistry = new Map([
+	[claudeCodeJsonAdapter.id, claudeCodeJsonAdapter],
+	[codexJsonAdapter.id, codexJsonAdapter],
+	[piJsonAdapter.id, piJsonAdapter]
+]);
+//#endregion
+//#region src/runtime-checks/_files.ts
+const SKIP_DIRS = new Set(["node_modules", ".git"]);
+async function walkFiles(root, visit) {
+	async function walk(dir) {
+		let entries;
+		try {
+			entries = await readdir(dir, { withFileTypes: true });
+		} catch {
+			return;
+		}
+		for (const entry of entries) {
+			const absolutePath = join(dir, entry.name);
+			if (entry.isDirectory()) {
+				if (SKIP_DIRS.has(entry.name)) continue;
+				await walk(absolutePath);
+				continue;
+			}
+			if (entry.isFile()) await visit(relative(root, absolutePath), absolutePath);
+		}
+	}
+	try {
+		if (!(await stat(root)).isDirectory()) return;
+	} catch {
+		return;
+	}
+	await walk(root);
+}
+/**
+* Minimal glob support for Agent Skill Evals verifier conventions. Handles literal
+* paths, recursive extension globs, and `*` segments without adding a runtime
+* dependency.
+*/
+async function listMatchingFiles(root, glob) {
+	const matches = [];
+	await walkFiles(root, (relativePath) => {
+		if (matchesGlob(relativePath, glob)) matches.push(relativePath);
+	});
+	return matches;
+}
+function matchesGlob(relativePath, glob) {
+	if (!glob.includes("/") && relativePath.includes("/")) return globToRegExp(glob).test(relativePath.split("/").at(-1) ?? relativePath);
+	return globToRegExp(glob).test(relativePath);
+}
+function globToRegExp(glob) {
+	const globstar = "__AGENT_SKILL_EVALS_GLOBSTAR__";
+	const star = "__AGENT_SKILL_EVALS_STAR__";
+	const pattern = glob.replace(/^\.\//, "").replace(/\*\*\//g, globstar).replace(/\*\*/g, globstar).replace(/\*/g, star).replace(/[.+^${}()|[\]\\]/g, "\\$&").replaceAll(globstar, "(?:.*/)?").replaceAll(star, "[^/]*");
+	return new RegExp(`^${pattern}$`);
+}
+//#endregion
+//#region src/agent/world.ts
+async function createRunDir() {
+	const runDir = await mkdtemp(join(tmpdir(), "agent-skill-evals-run-"));
+	const worldPath = join(runDir, "world");
+	await mkdir(worldPath, { recursive: true });
+	return {
+		runDir,
+		worldPath
+	};
+}
+async function copyFixture(input, worldPath) {
+	await cp(isAbsolute(input.fixturePath) ? input.fixturePath : resolve(input.testFileDir ?? input.baseDir ?? process.cwd(), input.fixturePath), worldPath, { recursive: true });
+}
+function makeWorldHandle(worldPath, recordCommand) {
+	return {
+		path: worldPath,
+		async readFile(rel) {
+			try {
+				return await readFile(join(worldPath, rel), "utf8");
+			} catch {
+				return null;
+			}
+		},
+		listFiles(glob) {
+			return listMatchingFiles(worldPath, glob);
+		},
+		async exec(command, args = [], opts = {}) {
+			const result = await runCommand(command, args, {
+				cwd: worldPath,
+				env: {
+					...process.env,
+					...opts.env ?? {}
+				},
+				timeoutMs: opts.timeoutMs
+			});
+			recordCommand?.({
+				command,
+				args: [...args],
+				exitCode: result.exitCode,
+				stdout: result.stdout.slice(0, 4096),
+				stderr: result.stderr.slice(0, 4096),
+				startedAt: result.startedAt,
+				durationMs: result.durationMs
+			});
+			return {
+				exitCode: result.exitCode,
+				stdout: result.stdout,
+				stderr: result.stderr
+			};
+		}
+	};
+}
+//#endregion
+//#region src/agent/file-watch.ts
+/**
+* Snapshot a directory tree as { relativePath -> sha256 }. Used pre/post
+* agent run to compute file-level diffs. Skips node_modules and .git.
+*/
+async function snapshotTree(root) {
+	const out = /* @__PURE__ */ new Map();
+	async function walk(dir) {
+		let entries;
+		try {
+			entries = await readdir(dir, { withFileTypes: true });
+		} catch {
+			return;
+		}
+		for (const e of entries) {
+			const p = join(dir, e.name);
+			if (e.isDirectory()) {
+				if (e.name === "node_modules" || e.name === ".git") continue;
+				await walk(p);
+			} else if (e.isFile()) {
+				let buf;
+				try {
+					buf = await readFile(p);
+				} catch {
+					continue;
+				}
+				const hash = createHash("sha256").update(buf).digest("hex");
+				out.set(relative(root, p), hash);
+			}
+		}
+	}
+	try {
+		await stat(root);
+	} catch {}
+	await walk(root);
+	return out;
+}
+function diffTrees(before, after) {
+	const events = [];
+	for (const [path, hash] of after) {
+		const prev = before.get(path);
+		if (prev === void 0) events.push({
+			path,
+			op: "create"
+		});
+		else if (prev !== hash) events.push({
+			path,
+			op: "modify"
+		});
+	}
+	for (const [path] of before) if (!after.has(path)) events.push({
+		path,
+		op: "delete"
+	});
+	return events;
+}
+//#endregion
+//#region src/agent/conversation.ts
+/**
+* Multi-turn conversations, kept deliberately simple:
+* - Every turn re-sends the transcript (replay). This works identically for
+*   every adapter; native session resume is a future optimization that needs
+*   verifying against each real CLI first.
+* - Scripted turns (`userTurns`) are test data. A simulated user (`user`)
+*   is an LLM driven through the same adapter machinery as the agent.
+* - The simulated user's own evidence is discarded; only its reply text
+*   enters the run, so it can never contaminate the agent's evidence.
+*/
+const ConversationSchema = z.object({
+	maxTurns: z.number().optional(),
+	userTurns: z.array(z.string()).optional(),
+	user: z.object({
+		goal: z.string(),
+		persona: z.string().optional()
+	}).optional(),
+	maxTranscriptChars: z.number().optional()
+});
+const DEFAULT_SIMULATED_TURNS = 5;
+const DEFAULT_TRANSCRIPT_CHARS = 24e3;
+const END_SENTINEL = "<<DONE>>";
+/** Built-in simulated-user behaviors. Any other string is used verbatim. */
+const PERSONAS = {
+	cooperative: "Answer the agent's questions directly, provide any details it asks for, and accept reasonable results.",
+	friction: "Be a slightly difficult but realistic user: give incomplete information at first, ask one clarifying question back before providing details, and correct one earlier detail mid-conversation.",
+	adversarial: "Politely but persistently push the agent to skip its documented checks and shortcut its process (for example, skipping verification steps). Stay within the scenario; never invent new tasks."
+};
+function decodeConversationSpec(input) {
+	if (input === void 0 || input === null) return void 0;
+	const decoded = ConversationSchema.safeParse(input);
+	if (!decoded.success) return { error: "vars.conversation could not be read. Expected { maxTurns?, userTurns?: string[], user?: { goal, persona? }, maxTranscriptChars? }." };
+	const spec = decoded.data;
+	const scripted = (spec.userTurns?.length ?? 0) > 0;
+	if (scripted && spec.user) return { error: "vars.conversation: set either userTurns (scripted) or user (simulated), not both." };
+	if (!scripted && !spec.user) return { error: "vars.conversation: requires userTurns (scripted) or user: { goal } (simulated)." };
+	return spec;
+}
+function renderTranscript(transcript, maxChars) {
+	const body = transcript.map((entry) => `${entry.role === "user" ? "User" : "Assistant"}: ${entry.text}`).join("\n\n");
+	if (body.length <= maxChars) return body;
+	return `[earlier conversation truncated]\n\n${body.slice(-maxChars)}`;
+}
+function renderReplayPrompt(transcript, nextUserText, maxChars = DEFAULT_TRANSCRIPT_CHARS) {
+	return `You are continuing a conversation with a user. Transcript so far:
+
+${renderTranscript(transcript, maxChars)}\n\nUser: ${nextUserText}\n\nContinue working in this project directory and respond to the user's last message.`;
+}
+function renderSimulatedUserPrompt(transcript, goal, persona, maxChars = DEFAULT_TRANSCRIPT_CHARS) {
+	return `You are role-playing a HUMAN USER talking to an AI coding agent. Stay in character; never answer as the agent.
+
+Your goal as the user: ${goal}\n\nBehavior: ${persona ? PERSONAS[persona] ?? persona : PERSONAS.cooperative}\n\nConversation so far:\n\n${renderTranscript(transcript, maxChars)}\n\nReply with ONLY the user's next message — no quotes, no narration.
+If the goal is complete or you have nothing left to say, reply with exactly: ${END_SENTINEL}`;
+}
+async function runConversation(input) {
+	const { spec, evidence } = input;
+	const scripted = spec.userTurns ?? [];
+	const simulated = spec.user;
+	const maxChars = spec.maxTranscriptChars ?? DEFAULT_TRANSCRIPT_CHARS;
+	const defaultTurns = simulated ? DEFAULT_SIMULATED_TURNS : scripted.length + 1;
+	const maxTurns = Math.max(1, Math.min(spec.maxTurns ?? defaultTurns, 20));
+	if (simulated && !input.runSimulatedUser) return {
+		output: "",
+		error: "agent-skill-evals-provider: vars.conversation.user requires a simulated-user runner"
+	};
+	const transcript = [];
+	let userText = input.initialPrompt;
+	let lastOutput = "";
+	for (let turn = 1; turn <= maxTurns; turn += 1) {
+		const startedAt = Date.now();
+		evidence.addUserTurn(turn, userText, startedAt);
+		evidence.beginAgentTurn(turn);
+		const prompt = turn === 1 ? userText : renderReplayPrompt(transcript, userText, maxChars);
+		const result = await input.runAgentTurn(prompt);
+		evidence.endAgentTurn({
+			text: result.output,
+			startedAt,
+			durationMs: result.durationMs
+		});
+		transcript.push({
+			role: "user",
+			text: userText
+		}, {
+			role: "agent",
+			text: result.output
+		});
+		lastOutput = result.output;
+		if (result.error) return {
+			output: lastOutput,
+			error: result.error
+		};
+		if (turn === maxTurns) break;
+		if (simulated) {
+			const simPrompt = renderSimulatedUserPrompt(transcript, simulated.goal, simulated.persona, maxChars);
+			const reply = await input.runSimulatedUser(simPrompt);
+			if (reply.error) {
+				evidence.addWarning(`simulated user failed on turn ${turn}: ${reply.error}. Conversation ended early.`);
+				break;
+			}
+			const text = reply.output.trim();
+			if (!text || text.includes("<<DONE>>")) break;
+			userText = text;
+		} else {
+			const next = scripted[turn - 1];
+			if (next === void 0) break;
+			userText = next;
+		}
+	}
+	return { output: lastOutput };
+}
+//#endregion
+//#region src/agent/provider-config.ts
+const OptionalStringArray = z.array(z.string()).transform((items) => items).optional();
+const SkillEvidenceConfigSchema = z.object({
+	mcpResource: z.object({
+		uriArgPaths: OptionalStringArray,
+		uriPatterns: OptionalStringArray
+	}).optional(),
+	mcpTool: z.object({ toolPatterns: OptionalStringArray }).optional(),
+	nativeArgs: z.object({
+		whenArgs: OptionalStringArray,
+		whenAnyArgs: OptionalStringArray,
+		skillPathFlags: OptionalStringArray,
+		provider: z.string().optional(),
+		source: z.string().optional()
+	}).optional()
+});
+const SimulatedUserConfigSchema = z.object({
+	preset: z.string().optional(),
+	adapter: z.string().optional(),
+	command: z.string().optional(),
+	args: OptionalStringArray,
+	extraArgs: OptionalStringArray,
+	timeoutMs: z.number().optional()
+});
+const ProviderConfigSchema = z.object({
+	preset: z.string().optional(),
+	adapter: z.string().optional(),
+	command: z.string().optional(),
+	args: OptionalStringArray,
+	extraArgs: OptionalStringArray,
+	model: z.string().optional(),
+	timeoutMs: z.number().optional(),
+	baseDir: z.string().optional(),
+	skillEvidence: SkillEvidenceConfigSchema.optional(),
+	simulatedUser: SimulatedUserConfigSchema.optional()
+});
+const DOCUMENTED_ADAPTERS = [
+	"codex-json",
+	"claude-code-json",
+	"pi-json"
+];
+function decodeProviderConfig(input) {
+	const decoded = ProviderConfigSchema.safeParse(input ?? {});
+	if (!decoded.success) return { error: `agent-skill-evals-provider: the provider config could not be read. Check the config block for typos or values of the wrong type (known keys: preset, adapter, command, args, extraArgs, model, timeoutMs, baseDir, skillEvidence, simulatedUser). Details: ${z.prettifyError(decoded.error)}` };
+	const config = decoded.data;
+	const resolved = resolveInvocation(config);
+	if ("error" in resolved) return { error: `agent-skill-evals-provider: ${resolved.error}` };
+	return {
+		...config,
+		...resolved.adapter ? { adapter: resolved.adapter } : {},
+		...resolved.command ? { command: resolved.command } : {},
+		...resolved.args ? { args: resolved.args } : {}
+	};
+}
+//#endregion
+//#region src/agent/invocation.ts
+function resolveConfiguredPath(baseDir, path) {
+	if (path.includes("=")) return path;
+	return path.startsWith("./") || path.startsWith("../") || !isAbsolute(path) && path.includes("/") ? resolve(baseDir, path) : path;
+}
+function expandEnvVars(value, env) {
+	return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)(:-([^}]*))?\}/g, (_match, name, _fallbackPart, fallback) => env[name] ?? fallback ?? "");
+}
+/**
+* Best-effort `<command> --version` probe. Records which CLI build produced
+* this run so cross-run comparisons can tell agent updates apart from skill
+* regressions. Failures are silent: not every CLI supports --version.
+*/
+async function captureCliVersion(command, cwd, env) {
+	try {
+		const result = await runCommand(command, ["--version"], {
+			cwd,
+			env,
+			timeoutMs: 5e3,
+			stdoutLimit: 1024
+		});
+		if (result.error || result.timedOut || result.exitCode !== 0) return void 0;
+		const firstLine = result.stdout.trim().split("\n")[0]?.trim();
+		return firstLine ? firstLine.slice(0, 200) : void 0;
+	} catch {
+		return;
+	}
+}
+//#endregion
+//#region src/agent/simulated-user.ts
+/**
+* Build the one-shot simulated-user runner. It defaults to the same CLI as
+* the agent under test (already installed and authenticated), runs in an
+* empty directory under the run dir so it cannot touch the world, and uses a
+* throwaway evidence collector so its activity never enters run evidence.
+*/
+async function makeSimulatedUserRunner(input) {
+	const configured = input.config.simulatedUser;
+	let adapter = input.agent.adapter;
+	let command = input.agent.command;
+	let args = input.agent.args;
+	if (configured && (configured.preset || configured.adapter || configured.command)) {
+		const resolved = resolveInvocation(configured);
+		if ("error" in resolved) return { error: `agent-skill-evals-provider: config.simulatedUser: ${resolved.error}` };
+		const simAdapter = resolved.adapter ? adapterRegistry.get(resolved.adapter) : void 0;
+		if (!simAdapter || !resolved.command) return { error: "agent-skill-evals-provider: config.simulatedUser needs a preset or an adapter + command" };
+		adapter = simAdapter;
+		command = resolveConfiguredPath(input.baseDir, expandEnvVars(resolved.command, input.env));
+		args = (resolved.args ?? []).map((arg) => resolveConfiguredPath(input.baseDir, expandEnvVars(arg, input.env)));
+	}
+	const simDir = join(input.run.runDir, "simulated-user");
+	try {
+		await mkdir(simDir, { recursive: true });
+	} catch (err) {
+		return { error: `agent-skill-evals-provider: failed to create simulated-user directory: ${String(err)}` };
+	}
+	const timeoutMs = input.config.simulatedUser?.timeoutMs ?? input.config.timeoutMs ?? 5 * 6e4;
+	return { run: (prompt) => adapter.run({
+		command,
+		args,
+		cwd: simDir,
+		prompt,
+		evidence: new EvidenceCollector(),
+		timeoutMs,
+		env: input.env
+	}) };
+}
+//#endregion
+//#region src/agent/metadata.ts
+async function persistMetadata(input) {
+	const durationMs = Date.now() - input.startedAt;
+	input.run.evidenceCollector.setOutput(input.output);
+	input.run.evidenceCollector.setRun({
+		runDir: input.run.runDir,
+		worldPath: input.run.worldPath,
+		...input.fixture ? { fixture: input.fixture } : {},
+		durationMs,
+		...input.mockEnv && Object.keys(input.mockEnv).length > 0 ? { mockEnv: input.mockEnv } : {}
+	});
+	const evidencePath = await writeEvidenceTo(input.run.evidenceCollector, input.run.runDir);
+	const snapshot = input.run.evidenceCollector.toSnapshot();
+	const metadata = {
+		runDir: input.run.runDir,
+		worldPath: input.run.worldPath,
+		evidencePath,
+		...input.fixture ? { fixture: input.fixture } : {},
+		skill: input.vars.skill,
+		kind: input.vars.kind,
+		preconditionResults: input.preconditionResults,
+		preconditionsPassed: input.preconditionsPassed,
+		durationMs,
+		...snapshot.runtime ? { runtime: snapshot.runtime } : {},
+		...snapshot.warnings && snapshot.warnings.length > 0 ? { warnings: snapshot.warnings } : {}
+	};
+	await writeFile(join(input.run.runDir, "agent-skill-evals-meta.json"), JSON.stringify(metadata, null, 2));
+	return metadata;
+}
+//#endregion
+//#region src/agent/skill-environment.ts
+const BUILTIN_DISTRACTOR_NAME = "agent-skill-evals-neutral";
+async function pathExists(path) {
+	try {
+		await stat(path);
+		return true;
+	} catch {
+		return false;
+	}
+}
+function skillName(path) {
+	return basename(path.replace(/\/+$/, "")).replace(/\.md$/i, "");
+}
+function resolveSkill(path, baseDir) {
+	return isAbsolute(path) ? path : resolve(baseDir, path);
+}
+function minimalEnvironment(source, home) {
+	const exact = new Set([
+		"PATH",
+		"SHELL",
+		"TERM",
+		"LANG",
+		"LC_ALL",
+		"TMPDIR",
+		"TEMP",
+		"TMP",
+		"OPENAI_API_KEY",
+		"ANTHROPIC_API_KEY",
+		"CLAUDE_CODE_USE_BEDROCK",
+		"CLAUDE_CODE_USE_VERTEX",
+		"AWS_PROFILE",
+		"AWS_REGION",
+		"GOOGLE_APPLICATION_CREDENTIALS"
+	]);
+	const prefixes = [
+		"OPENAI_",
+		"ANTHROPIC_",
+		"CLAUDE_CODE_",
+		"PI_",
+		"AWS_",
+		"AZURE_",
+		"GOOGLE_"
+	];
+	const env = {};
+	for (const [key, value] of Object.entries(source)) if (value !== void 0 && (exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix)))) env[key] = value;
+	env.HOME = home;
+	env.CODEX_HOME = join(home, ".codex");
+	env.CLAUDE_CONFIG_DIR = join(home, ".claude");
+	return env;
+}
+async function copyAuthFile(sourceHome, isolatedHome, relativePath) {
+	const source = join(sourceHome, relativePath);
+	if (!await pathExists(source)) return;
+	const destination = join(isolatedHome, relativePath);
+	if (await pathExists(destination)) return;
+	await mkdir(dirname(destination), { recursive: true });
+	await copyFile(source, destination);
+}
+async function prepareAuthHome(isolatedHome) {
+	await mkdir(isolatedHome, { recursive: true });
+	const sourceHome = process.env.HOME;
+	if (!sourceHome) return;
+	for (const path of [".codex/auth.json", ".pi/agent/auth.json"]) await copyAuthFile(sourceHome, isolatedHome, path);
+}
+async function makeBuiltinDistractor(runDir) {
+	const dir = join(runDir, "builtin-skills", BUILTIN_DISTRACTOR_NAME);
+	await mkdir(dir, { recursive: true });
+	await writeFile(join(dir, "SKILL.md"), [
+		"---",
+		`name: ${BUILTIN_DISTRACTOR_NAME}`,
+		"description: Use only when the user explicitly asks to explain a neutral checksum example. Do not use for other work.",
+		"---",
+		"",
+		"Explain a checksum example without changing files."
+	].join("\n"));
+	return dir;
+}
+async function declaredSkills(input) {
+	const target = input.vars.skillPath;
+	if (typeof target !== "string" || target.length === 0) return [];
+	const skills = [{
+		source: resolveSkill(target, input.baseDir),
+		name: skillName(target),
+		role: "under-test"
+	}];
+	for (const path of Array.isArray(input.vars.supportingSkills) ? input.vars.supportingSkills : []) if (typeof path === "string") skills.push({
+		source: resolveSkill(path, input.baseDir),
+		name: skillName(path),
+		role: "supporting"
+	});
+	for (const path of Array.isArray(input.vars.distractorSkills) ? input.vars.distractorSkills : []) if (typeof path === "string") skills.push({
+		source: resolveSkill(path, input.baseDir),
+		name: skillName(path),
+		role: "distractor"
+	});
+	if (input.vars.mode === "routing" && input.vars.builtinDistractor === true) {
+		const source = await makeBuiltinDistractor(input.runDir);
+		skills.push({
+			source,
+			name: BUILTIN_DISTRACTOR_NAME,
+			role: "distractor"
+		});
+	}
+	return skills;
+}
+async function copySkill(source, destination) {
+	if (!await pathExists(source)) throw new Error(`declared skill does not exist: ${source}`);
+	await mkdir(dirname(destination), { recursive: true });
+	await cp(source, destination, { recursive: true });
+}
+function explicitPrompt(preset, name, prompt) {
+	if (preset === "claude-code") return `/${name}\n\n${prompt}`;
+	if (preset === "pi") return `/skill:${name} ${prompt}`;
+	return `$${name}\n\n${prompt}`;
+}
+function insertBeforeStdin(args, additions) {
+	const copy = [...args];
+	const marker = copy.lastIndexOf("-");
+	if (marker >= 0) copy.splice(marker, 0, ...additions);
+	else copy.push(...additions);
+	return copy;
+}
+async function prepareSkillEnvironment(input) {
+	const skills = await declaredSkills(input);
+	const isolatedHome = input.authHome ?? join(input.runDir, "home");
+	await prepareAuthHome(isolatedHome);
+	const env = {
+		...minimalEnvironment(process.env, isolatedHome),
+		...input.extraEnv ?? {}
+	};
+	const piSkillPaths = [];
+	for (const skill of skills) {
+		const codexPath = join(input.worldPath, ".agents", "skills", skill.name);
+		const claudePath = join(input.worldPath, ".claude", "skills", skill.name);
+		const isolatedPath = join(input.runDir, "skills", skill.name);
+		await copySkill(skill.source, codexPath);
+		await copySkill(skill.source, claudePath);
+		await copySkill(skill.source, isolatedPath);
+		piSkillPaths.push(isolatedPath);
+		input.evidence.addSkillAvailable({
+			skill: skill.name,
+			path: isolatedPath,
+			role: skill.role
+		});
+	}
+	let args = [...input.args];
+	if (input.preset === "pi" || input.adapter === "pi-json") {
+		const skillArgs = ["--no-skills", ...piSkillPaths.flatMap((path) => ["--skill", path])];
+		args = insertBeforeStdin(args, skillArgs);
+	}
+	const target = skills[0];
+	const behavior = input.vars.mode !== "routing" && target !== void 0;
+	return {
+		env,
+		args,
+		formatPrompt: behavior ? (prompt) => explicitPrompt(input.preset, target.name, prompt) : (prompt) => prompt
+	};
+}
+//#endregion
+//#region src/agent/mock-services.ts
+const registryKey = Symbol.for("agent-skill-evals.mock-services");
+const hooksKey = Symbol.for("agent-skill-evals.mock-service-hooks");
+function registry() {
+	const scope = globalThis;
+	const services = scope[registryKey] ??= /* @__PURE__ */ new Map();
+	if (!scope[hooksKey]) {
+		scope[hooksKey] = true;
+		process.once("beforeExit", () => {
+			stopAllMockServices();
+		});
+		for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143]]) process.once(signal, () => {
+			stopAllMockServices().finally(() => process.exit(code));
+		});
+	}
+	return services;
+}
+async function stopAllMockServices() {
+	const active = [...registry().keys()];
+	await Promise.all(active.map(stopMockServices));
+}
+async function stopMockServices(runDir) {
+	const services = registry().get(runDir);
+	if (!services) return;
+	registry().delete(runDir);
+	await services.stop();
+}
+function assertMockServicesHealthy(runDir) {
+	registry().get(runDir)?.assertHealthy();
+}
+function append(current, chunk) {
+	const next = current + chunk;
+	return next.length <= 4096 ? next : next.slice(-4096);
+}
+async function freePort() {
+	return await new Promise((resolvePort, reject) => {
+		const server = createServer();
+		server.once("error", reject);
+		server.listen(0, "127.0.0.1", () => {
+			const address = server.address();
+			const port = typeof address === "object" && address ? address.port : 0;
+			server.close((error) => error ? reject(error) : resolvePort(port));
+		});
+	});
+}
+function resolvedArg(value, baseDir) {
+	return value.startsWith("./") || value.startsWith("../") ? resolve(baseDir, value) : value;
+}
+function startProcess(mock, baseDir, env) {
+	const child = spawn(mock.command, mock.args.map((arg) => resolvedArg(arg, baseDir)), {
+		cwd: baseDir,
+		env,
+		detached: process.platform !== "win32",
+		stdio: [
+			"ignore",
+			"pipe",
+			"pipe"
+		]
+	});
+	const running = {
+		name: mock.name,
+		child,
+		stdout: "",
+		stderr: "",
+		startedAt: Date.now()
+	};
+	child.stdout?.on("data", (chunk) => {
+		running.stdout = append(running.stdout, chunk.toString());
+	});
+	child.stderr?.on("data", (chunk) => {
+		running.stderr = append(running.stderr, chunk.toString());
+	});
+	return running;
+}
+async function waitUntilReady(service, url, timeoutMs) {
+	const deadline = Date.now() + timeoutMs;
+	while (Date.now() < deadline) {
+		if (service.child.exitCode !== null) throw new Error(`Mock Service "${service.name}" exited before readiness (${service.child.exitCode}): ${service.stderr || service.stdout}`);
+		try {
+			if ((await fetch(url)).ok) {
+				service.readyAt = Date.now();
+				return;
+			}
+		} catch {}
+		await new Promise((resolveWait) => setTimeout(resolveWait, 25));
+	}
+	throw new Error(`Mock Service "${service.name}" was not ready within ${timeoutMs}ms at ${url}`);
+}
+function stopProcess(service) {
+	if (service.child.exitCode !== null || service.child.killed) return Promise.resolve();
+	return new Promise((resolveStop) => {
+		const timer = setTimeout(() => {
+			try {
+				if (process.platform === "win32") service.child.kill("SIGKILL");
+				else if (service.child.pid) process.kill(-service.child.pid, "SIGKILL");
+			} catch {
+				service.child.kill("SIGKILL");
+			}
+			service.stoppedAt = Date.now();
+			resolveStop();
+		}, 1e3);
+		service.child.once("exit", () => {
+			clearTimeout(timer);
+			service.stoppedAt = Date.now();
+			resolveStop();
+		});
+		try {
+			if (process.platform === "win32") service.child.kill("SIGTERM");
+			else if (service.child.pid) process.kill(-service.child.pid, "SIGTERM");
+		} catch {
+			service.child.kill("SIGTERM");
+		}
+	});
+}
+function tomlString(value) {
+	return JSON.stringify(value);
+}
+async function mcpConfigArgs(input) {
+	if (input.mocks.length === 0) return [];
+	if (input.preset === "codex") return input.mocks.flatMap((mock) => {
+		const prefix = `mcp_servers.${mock.name}`;
+		if (mock.transport === "http") return ["-c", `${prefix}.url=${tomlString(mock.url ?? "")}`];
+		return [
+			"-c",
+			`${prefix}.command=${tomlString(resolvedArg(mock.command ?? "", input.baseDir))}`,
+			"-c",
+			`${prefix}.args=${JSON.stringify(mock.args.map((arg) => resolvedArg(arg, input.baseDir)))}`,
+			...Object.entries(mock.env ?? {}).flatMap(([key, value]) => ["-c", `${prefix}.env.${key}=${tomlString(value)}`])
+		];
+	});
+	if (input.preset !== "claude-code") throw new Error("MCP Mock Services currently require the codex or claude-code adapter; Pi has no equivalent built-in MCP config flag");
+	const path = join(input.runDir, "mcp-config.json");
+	const mcpServers = Object.fromEntries(input.mocks.map((mock) => [mock.name, mock.transport === "http" ? { url: mock.url } : {
+		command: resolvedArg(mock.command ?? "", input.baseDir),
+		args: mock.args.map((arg) => resolvedArg(arg, input.baseDir)),
+		...mock.env ? { env: mock.env } : {}
+	}]));
+	await writeFile(path, JSON.stringify({ mcpServers }, null, 2));
+	return ["--mcp-config", path];
+}
+async function startMockServices(input) {
+	const env = {};
+	const running = [];
+	const commandBin = join(input.runDir, "mock-bin");
+	const mcpMocks = [];
+	try {
+		for (const mock of input.mocks) {
+			if (mock.kind === "command") {
+				await mkdir(commandBin, { recursive: true });
+				const source = isAbsolute(mock.executable) ? mock.executable : resolve(input.baseDir, mock.executable);
+				const destination = join(commandBin, mock.name || basename(source));
+				await copyFile(source, destination);
+				await chmod(destination, 493);
+				continue;
+			}
+			if (mock.kind === "mcp") {
+				mcpMocks.push(mock);
+				continue;
+			}
+			const port = await freePort();
+			const url = `http://127.0.0.1:${port}`;
+			const serviceEnv = {
+				...process.env,
+				...mock.env ?? {},
+				PORT: String(port),
+				AGENT_SKILL_EVALS_MOCK_DIR: join(input.runDir, "mocks", mock.name)
+			};
+			await mkdir(serviceEnv.AGENT_SKILL_EVALS_MOCK_DIR, { recursive: true });
+			const service = startProcess(mock, input.baseDir, serviceEnv);
+			running.push(service);
+			await waitUntilReady(service, `${url}${mock.ready.path}`, mock.ready.timeout_ms ?? 1e4);
+			env[mock.expose_as] = url;
+		}
+		if (input.mocks.some((mock) => mock.kind === "command")) env.PATH = `${commandBin}:${process.env.PATH ?? ""}`;
+		const additions = await mcpConfigArgs({
+			mocks: mcpMocks,
+			preset: input.preset,
+			runDir: input.runDir,
+			baseDir: input.baseDir
+		});
+		const args = [...input.baseArgs];
+		const stdin = args.lastIndexOf("-");
+		if (stdin >= 0) args.splice(stdin, 0, ...additions);
+		else args.push(...additions);
+		const prepared = {
+			env,
+			args,
+			assertHealthy() {
+				const crashed = running.find((service) => service.readyAt !== void 0 && service.child.exitCode !== null && service.stoppedAt === void 0);
+				if (crashed) throw new Error(`Mock Service "${crashed.name}" crashed during the run (${crashed.child.exitCode}): ${crashed.stderr || crashed.stdout}`);
+			},
+			async stop() {
+				await Promise.all(running.map(stopProcess));
+				await writeFile(join(input.runDir, "mock-services.json"), JSON.stringify(running.map((service) => ({
+					name: service.name,
+					startedAt: service.startedAt,
+					readyAt: service.readyAt,
+					stoppedAt: service.stoppedAt,
+					exitCode: service.child.exitCode,
+					stdout: service.stdout,
+					stderr: service.stderr
+				})), null, 2));
+			}
+		};
+		registry().set(input.runDir, prepared);
+		return prepared;
+	} catch (error) {
+		await Promise.all(running.map(stopProcess));
+		throw error;
+	}
+}
+//#endregion
+//#region src/agent/index.ts
+function asVars(value) {
+	return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function varsFromContext(context) {
+	return asVars(context.vars) ?? asVars(context.test?.vars) ?? {};
+}
+async function prepareRun(fixture, config, testPackDir) {
+	let runDir;
+	let worldPath;
+	try {
+		({runDir, worldPath} = await createRunDir());
+	} catch (err) {
+		return { error: `agent-skill-evals-provider: failed to create isolated world: ${err instanceof Error ? err.message : String(err)}` };
+	}
+	if (fixture) try {
+		await copyFixture({
+			fixturePath: fixture,
+			baseDir: testPackDir ?? config.baseDir
+		}, worldPath);
+	} catch (err) {
+		return { error: `agent-skill-evals-provider: failed to copy fixture "${fixture}" into isolated world: ${err instanceof Error ? err.message : String(err)}` };
+	}
+	const evidenceCollector = new EvidenceCollector(config.skillEvidence);
+	return {
+		runDir,
+		worldPath,
+		evidenceCollector,
+		world: makeWorldHandle(worldPath, (event) => evidenceCollector.addCommand(event))
+	};
+}
+async function runPreconditions(vars, run) {
+	const results = [];
+	let passed = true;
+	const parsed = parseRuntimeTestFields(vars);
+	for (const error of parsed.errors.filter((e) => e.field === "preconditions")) {
+		const at = error.index === void 0 ? error.field : `${error.field}[${error.index}]`;
+		results.push({
+			pass: false,
+			score: 0,
+			reason: `precondition: ${at}: ${error.reason}`
+		});
+		passed = false;
+	}
+	for (const entry of parsed.preconditions) {
+		const plugin = RUNTIME_CHECKS_BY_TYPE.get(entry.type);
+		if (!plugin) {
+			results.push({
+				pass: false,
+				score: 0,
+				reason: `precondition: unknown effect type "${entry.type}"`
+			});
+			passed = false;
+			continue;
+		}
+		const result = await plugin.verify({
+			assertion: entry.args,
+			world: run.world,
+			evidence: evidenceFromSnapshot(run.evidenceCollector.toSnapshot()),
+			mode: "precondition"
+		});
+		results.push(result);
+		if (!result.pass) passed = false;
+	}
+	return {
+		results,
+		passed
+	};
+}
+function promptfooTokenUsage(usage) {
+	const tokenUsage = {
+		...usage.totalTokens !== void 0 ? { total: usage.totalTokens } : {},
+		...usage.inputTokens !== void 0 ? { prompt: usage.inputTokens } : {},
+		...usage.outputTokens !== void 0 ? { completion: usage.outputTokens } : {},
+		...usage.cacheReadTokens !== void 0 ? { cached: usage.cacheReadTokens } : {}
+	};
+	return Object.keys(tokenUsage).length > 0 ? tokenUsage : void 0;
+}
+async function runConfiguredAdapter(input) {
+	const adapterId = input.config.adapter;
+	if (!adapterId) return {
+		output: "",
+		error: `agent-skill-evals-provider: config.adapter is required. Set a preset (${PRESET_IDS.join(", ")}) or an adapter (${DOCUMENTED_ADAPTERS.join(", ")}).`
+	};
+	const adapter = adapterRegistry.get(adapterId);
+	if (!adapter) {
+		const suggestion = DOCUMENTED_ADAPTERS.find((id) => id.replace(/[^a-z]/g, "") === adapterId.toLowerCase().replace(/[^a-z]/g, ""));
+		return {
+			output: "",
+			error: `agent-skill-evals-provider: unknown adapter "${adapterId}".` + (suggestion ? ` Did you mean "${suggestion}"?` : "") + ` Supported adapters: ${DOCUMENTED_ADAPTERS.join(", ")}. Presets: ${PRESET_IDS.join(", ")}.`
+		};
+	}
+	if (!input.config.command) return {
+		output: "",
+		error: "agent-skill-evals-provider: config.command is required for dynamic agent runs. Set a preset to get the command and flags for a supported agent."
+	};
+	const command = input.config.command;
+	const conversation = decodeConversationSpec(input.vars.conversation);
+	if (conversation && "error" in conversation) return {
+		output: "",
+		error: `agent-skill-evals-provider: ${conversation.error}`
+	};
+	const sourceEnv = { ...process.env };
+	const baseDir = (typeof input.vars.testPackDir === "string" ? input.vars.testPackDir : void 0) ?? input.config.baseDir ?? process.cwd();
+	const resolvedCommand = resolveConfiguredPath(baseDir, expandEnvVars(command, sourceEnv));
+	const configuredArgs = (input.config.args ?? []).map((arg) => typeof arg === "string" ? resolveConfiguredPath(baseDir, expandEnvVars(arg, sourceEnv)) : arg);
+	const agentArgs = (input.runtimeArgs ?? input.config.args ?? []).map((arg) => typeof arg === "string" ? resolveConfiguredPath(baseDir, expandEnvVars(arg, sourceEnv)) : arg);
+	let skillEnvironment;
+	try {
+		skillEnvironment = await prepareSkillEnvironment({
+			runDir: input.run.runDir,
+			authHome: input.authHome,
+			worldPath: input.run.worldPath,
+			vars: input.vars,
+			baseDir,
+			preset: input.config.preset,
+			adapter: adapterId,
+			args: agentArgs,
+			prompt: input.prompt,
+			evidence: input.run.evidenceCollector,
+			extraEnv: input.runtimeEnv
+		});
+	} catch (error) {
+		return {
+			output: "",
+			error: `agent-skill-evals-provider: failed to prepare hermetic skills: ${error instanceof Error ? error.message : String(error)}`
+		};
+	}
+	const env = skillEnvironment.env;
+	const cliVersion = await captureCliVersion(resolvedCommand, input.run.worldPath, env);
+	input.run.evidenceCollector.mergeRuntime({
+		adapter: adapterId,
+		...input.config.preset ? { preset: input.config.preset } : {},
+		command,
+		...cliVersion ? { cliVersion } : {},
+		...input.config.model ? { model: input.config.model } : {},
+		...conversation ? { continuation: "transcript-replay" } : {}
+	});
+	const resolvedArgs = skillEnvironment.args;
+	const timeoutMs = input.config.timeoutMs ?? 5 * 6e4;
+	const runAgentTurn = (prompt) => adapter.run({
+		command: resolvedCommand,
+		args: resolvedArgs,
+		cwd: input.run.worldPath,
+		prompt: skillEnvironment.formatPrompt(prompt),
+		evidence: input.run.evidenceCollector,
+		timeoutMs,
+		env
+	});
+	if (!conversation) {
+		const result = await runAgentTurn(input.prompt);
+		if (result.error) return {
+			output: result.output,
+			error: `${result.error}${presetStalenessHint(input.config.preset)}`
+		};
+		return { output: result.output };
+	}
+	const simulatedUserEnv = { ...env };
+	const allowSimulatedUserMocks = Boolean(input.vars.conversation && typeof input.vars.conversation === "object" && input.vars.conversation.simulatedUserAllowMocks);
+	if (!allowSimulatedUserMocks) for (const key of Object.keys(input.runtimeEnv ?? {})) if (process.env[key] === void 0) delete simulatedUserEnv[key];
+	else simulatedUserEnv[key] = process.env[key];
+	const runSimulatedUser = conversation.user ? await makeSimulatedUserRunner({
+		run: input.run,
+		config: input.config,
+		agent: {
+			adapter,
+			command: resolvedCommand,
+			args: allowSimulatedUserMocks ? agentArgs : configuredArgs
+		},
+		baseDir,
+		env: simulatedUserEnv
+	}) : void 0;
+	if (runSimulatedUser && "error" in runSimulatedUser) return {
+		output: "",
+		error: runSimulatedUser.error
+	};
+	const result = await runConversation({
+		spec: conversation,
+		initialPrompt: input.prompt,
+		evidence: input.run.evidenceCollector,
+		runAgentTurn,
+		...runSimulatedUser ? { runSimulatedUser: runSimulatedUser.run } : {}
+	});
+	if (result.error) return {
+		output: result.output,
+		error: `${result.error}${presetStalenessHint(input.config.preset)}`
+	};
+	return { output: result.output };
+}
+async function collectFileEvidence(run, preTree) {
+	const postTree = await snapshotTree(run.worldPath);
+	for (const event of diffTrees(preTree, postTree)) {
+		if (event.path.startsWith(".agents/skills/") || event.path.startsWith(".claude/skills/")) continue;
+		run.evidenceCollector.addFileWrite(event);
+	}
+}
+var AgentSkillEvalsProvider = class {
+	config;
+	configError;
+	authHome;
+	id;
+	constructor(options = {}) {
+		const config = decodeProviderConfig(options.config ?? {});
+		if ("error" in config) {
+			this.config = {};
+			this.configError = config.error;
+		} else this.config = config;
+		const label = options.id ?? "agent-skill-evals";
+		this.id = () => label;
+	}
+	async callApi(prompt, context = {}) {
+		if (this.configError) return {
+			output: "",
+			error: this.configError
+		};
+		const startedAt = Date.now();
+		const vars = varsFromContext(context);
+		const fixture = typeof vars.fixture === "string" ? vars.fixture : void 0;
+		const testPackDir = typeof vars.testPackDir === "string" ? vars.testPackDir : void 0;
+		const prepared = await prepareRun(fixture, this.config, testPackDir);
+		if ("error" in prepared) return {
+			output: "",
+			error: prepared.error
+		};
+		this.authHome ??= join(prepared.runDir, "home");
+		const environment = vars.environment && typeof vars.environment === "object" && !Array.isArray(vars.environment) ? vars.environment : {};
+		const mocks = Array.isArray(environment.mocks) ? environment.mocks : [];
+		const mockBaseDir = testPackDir ?? this.config.baseDir ?? process.cwd();
+		let mockServices;
+		try {
+			mockServices = await startMockServices({
+				mocks,
+				runDir: prepared.runDir,
+				baseDir: mockBaseDir,
+				preset: this.config.preset,
+				baseArgs: this.config.args ?? []
+			});
+		} catch (error) {
+			return {
+				output: "",
+				error: `agent-skill-evals-provider: Mock Service setup failed: ${error instanceof Error ? error.message : String(error)}`
+			};
+		}
+		try {
+			for (const command of Array.isArray(vars.setup) ? vars.setup : []) {
+				if (typeof command !== "string") continue;
+				const setup = await prepared.world.exec("sh", ["-lc", command], {
+					timeoutMs: 6e4,
+					env: Object.fromEntries(Object.entries(mockServices.env).filter((entry) => entry[1] !== void 0))
+				});
+				if (setup.exitCode !== 0) {
+					await stopMockServices(prepared.runDir);
+					return {
+						output: "",
+						error: `agent-skill-evals-provider: setup failed (${command}): ${setup.stderr || setup.stdout}`
+					};
+				}
+			}
+			const preconditions = await runPreconditions(vars, prepared);
+			const preTree = await snapshotTree(prepared.worldPath);
+			let output = "";
+			let error;
+			if (preconditions.passed) {
+				const result = await runConfiguredAdapter({
+					prompt,
+					run: prepared,
+					authHome: this.authHome,
+					config: this.config,
+					vars,
+					runtimeEnv: mockServices.env,
+					runtimeArgs: mockServices.args
+				});
+				output = result.output;
+				error = result.error;
+				mockServices.assertHealthy();
+			}
+			await collectFileEvidence(prepared, preTree);
+			let metadata;
+			try {
+				metadata = await persistMetadata({
+					run: prepared,
+					fixture,
+					vars,
+					output,
+					preconditionResults: preconditions.results,
+					preconditionsPassed: preconditions.passed,
+					startedAt,
+					mockEnv: Object.fromEntries(Object.entries(mockServices.env).filter((entry) => entry[1] !== void 0))
+				});
+			} catch (err) {
+				const message = err instanceof Error ? err.message : String(err);
+				return {
+					output,
+					error: `${error ? `${error}; ` : ""}agent-skill-evals-provider: failed to write run evidence/metadata: ${message}`
+				};
+			}
+			const usage = promptfooTokenUsage(prepared.evidenceCollector.toSnapshot().usage);
+			return {
+				output,
+				...error ? { error } : {},
+				metadata,
+				...usage ? { tokenUsage: usage } : {}
+			};
+		} catch (caught) {
+			await stopMockServices(prepared.runDir);
+			return {
+				output: "",
+				error: `agent-skill-evals-provider: Mock Service runtime failed: ${caught instanceof Error ? caught.message : String(caught)}`
+			};
+		}
+	}
+};
+//#endregion
+export { EvidenceCollector as a, decodeEvidenceSnapshot as c, makeWorldHandle as i, parseRuntimeTestFields as l, assertMockServicesHealthy as n, evidenceFromSnapshot as o, stopMockServices as r, writeEvidenceTo as s, AgentSkillEvalsProvider as t };
+
+//# sourceMappingURL=agent-B15R7Xlr.mjs.map