agent-skill-evals 0.1.0

package/dist/agent-CM7fIL_C.mjs

@@ -0,0 +1,1525 @@
+import { c as parseRuntimeTestFields, n as FileSystem, r as NodeServicesLive, t as Environment } from "./internal-services-5-mRgNls.mjs";
+import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
+import * as Clock from "effect/Clock";
+import * as Either from "effect/Either";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as ParseResult from "effect/ParseResult";
+import * as Schema from "effect/Schema";
+import * as Context from "effect/Context";
+import { spawn } from "node:child_process";
+import "effect/Stream";
+import { createHash } from "node:crypto";
+//#region src/runtime-checks/_helpers.ts
+function result(pass, reason, evidence) {
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		evidence
+	};
+}
+function validationFailure(reason) {
+	return result(false, reason);
+}
+/**
+* Map a "matched" boolean to a pass result based on mode. Used by effect
+* types that don't self-encode polarity (file.exists, tool.called, ...).
+*
+* - should: pass = matched
+* - should_not: pass = !matched
+* - precondition: pass = matched (precondition asserts a current state)
+*/
+function applyMode(matched, mode, reasonMatched, reasonUnmatched) {
+	switch (mode) {
+		case "should":
+		case "precondition": return result(matched, matched ? reasonMatched : reasonUnmatched);
+		case "should_not": return result(!matched, matched ? reasonMatched : reasonUnmatched);
+	}
+}
+//#endregion
+//#region src/runtime-checks/schemas.ts
+const NonEmptyString = Schema.String.pipe(Schema.filter((value) => value.trim().length > 0, { identifier: "NonEmptyString" }));
+const OptionalString$1 = Schema.optional(Schema.String);
+const PathArgsSchema = Schema.Struct({ path: NonEmptyString });
+const FileContainsArgsSchema = Schema.Struct({
+	path: NonEmptyString,
+	text: Schema.String
+});
+const FileChangesOutsideScopeArgsSchema = Schema.Struct({ scope: Schema.Array(NonEmptyString).pipe(Schema.minItems(1)) });
+const CodePatternArgsSchema = Schema.Struct({
+	glob: NonEmptyString,
+	pattern: NonEmptyString
+});
+const VerifierArgsSchema = Schema.Struct({
+	run: NonEmptyString,
+	args: Schema.optional(Schema.Array(Schema.String)),
+	timeoutMs: Schema.optional(Schema.Number)
+});
+const ToolCalledArgsSchema = Schema.Struct({
+	tool: NonEmptyString,
+	provider: OptionalString$1,
+	server: OptionalString$1,
+	args_match: Schema.optional(Schema.Unknown)
+});
+const ToolNotCalledArgsSchema = Schema.Struct({
+	tool: OptionalString$1,
+	provider: OptionalString$1,
+	server: OptionalString$1,
+	args_match: Schema.optional(Schema.Unknown)
+});
+const SkillLoadedArgsSchema = Schema.Struct({
+	should_include: Schema.optional(Schema.Array(NonEmptyString)),
+	should_exclude: Schema.optional(Schema.Array(NonEmptyString)),
+	delivery: Schema.optional(Schema.Literal("native", "mcp")),
+	provider: OptionalString$1,
+	server: OptionalString$1,
+	source: OptionalString$1
+});
+function decodeCheckArgs(schema, assertion, invalidReason) {
+	const decoded = Schema.decodeUnknownEither(schema, { errors: "all" })(assertion ?? {});
+	return Either.isRight(decoded) ? decoded.right : validationFailure(invalidReason);
+}
+function isValidationFailure(value) {
+	return !!value && typeof value === "object" && "pass" in value && value.pass === false && typeof value.reason === "string";
+}
+function decodeToolNotCalledArgs(assertion) {
+	return decodeCheckArgs(ToolNotCalledArgsSchema, assertion, "tool.not_called: assertion must include at least one selector");
+}
+//#endregion
+//#region src/runtime-checks/verifier-succeeds.ts
+const verifierSucceeds = {
+	type: "verifier.succeeds",
+	verify(ctx) {
+		return Effect.gen(function* () {
+			const a = decodeCheckArgs(VerifierArgsSchema, ctx.assertion, "verifier.succeeds: assertion.run must be a non-empty string");
+			if (isValidationFailure(a)) return a;
+			const r = yield* ctx.world.exec(a.run, a.args ?? [], { timeoutMs: a.timeoutMs ?? 6e4 });
+			return applyMode(r.exitCode === 0, ctx.mode, `verifier.succeeds: ${a.run} exited 0`, `verifier.succeeds: ${a.run} exited ${r.exitCode}: ${r.stderr.slice(0, 200)}`);
+		});
+	}
+};
+//#endregion
+//#region src/runtime-checks/verifier-fails.ts
+const verifierFails = {
+	type: "verifier.fails",
+	verify(ctx) {
+		return Effect.gen(function* () {
+			const a = decodeCheckArgs(VerifierArgsSchema, ctx.assertion, "verifier.fails: assertion.run must be a non-empty string");
+			if (isValidationFailure(a)) return a;
+			const r = yield* ctx.world.exec(a.run, a.args ?? [], { timeoutMs: a.timeoutMs ?? 6e4 });
+			return applyMode(r.exitCode !== 0, ctx.mode, `verifier.fails: ${a.run} exited ${r.exitCode} (failed as expected)`, `verifier.fails: ${a.run} unexpectedly exited 0`);
+		});
+	}
+};
+//#endregion
+//#region src/runtime-checks/file-exists.ts
+const fileExists = {
+	type: "file.exists",
+	verify(ctx) {
+		return Effect.gen(function* () {
+			const a = decodeCheckArgs(PathArgsSchema, ctx.assertion, "file.exists: assertion.path must be a non-empty string");
+			if (isValidationFailure(a)) return a;
+			return applyMode((yield* ctx.world.readFile(a.path)) !== null, ctx.mode, `file.exists: ${a.path} present`, `file.exists: ${a.path} not found`);
+		});
+	}
+};
+//#endregion
+//#region src/runtime-checks/file-created.ts
+const fileCreated = {
+	type: "file.created",
+	verify(ctx) {
+		const a = decodeCheckArgs(PathArgsSchema, ctx.assertion, "file.created: assertion.path must be a non-empty string");
+		if (isValidationFailure(a)) return Effect.succeed(a);
+		const created = ctx.evidence.filesWritten().some((ev) => ev.path === a.path && ev.op === "create");
+		return Effect.succeed(applyMode(created, ctx.mode, `file.created: ${a.path} created`, `file.created: ${a.path} was not created`));
+	}
+};
+//#endregion
+//#region src/runtime-checks/file-not-modified.ts
+const fileNotModified = {
+	type: "file.not_modified",
+	verify(ctx) {
+		const a = decodeCheckArgs(PathArgsSchema, ctx.assertion, "file.not_modified: assertion.path must be a non-empty string");
+		if (isValidationFailure(a)) return Effect.succeed(a);
+		const matched = !ctx.evidence.filesWritten().some((f) => f.path === a.path);
+		return Effect.succeed(applyMode(matched, ctx.mode, `file.not_modified: ${a.path} unchanged`, `file.not_modified: ${a.path} was modified`));
+	}
+};
+//#endregion
+//#region src/runtime-checks/file-contains.ts
+const fileContains = {
+	type: "file.contains",
+	verify(ctx) {
+		return Effect.gen(function* () {
+			const a = decodeCheckArgs(FileContainsArgsSchema, ctx.assertion, "file.contains: assertion.path must be a non-empty string and assertion.text must be a string");
+			if (isValidationFailure(a)) return a;
+			const content = yield* ctx.world.readFile(a.path);
+			return applyMode(content !== null && content.includes(a.text), ctx.mode, `file.contains: ${a.path} contains "${a.text.slice(0, 40)}"`, content === null ? `file.contains: ${a.path} not found` : `file.contains: ${a.path} missing "${a.text.slice(0, 40)}"`);
+		});
+	}
+};
+//#endregion
+//#region src/runtime-checks/code-pattern-exists.ts
+const codePatternExists = {
+	type: "code.pattern_exists",
+	verify(ctx) {
+		return Effect.gen(function* () {
+			const a = decodeCheckArgs(CodePatternArgsSchema, ctx.assertion, "code.pattern_exists: assertion.glob and assertion.pattern must be non-empty strings");
+			if (isValidationFailure(a)) return a;
+			let re;
+			try {
+				re = new RegExp(a.pattern);
+			} catch (err) {
+				return {
+					pass: false,
+					score: 0,
+					reason: `code.pattern_exists: invalid assertion.pattern regex: ${err instanceof Error ? err.message : String(err)}`
+				};
+			}
+			const files = yield* ctx.world.listFiles(a.glob);
+			const matchedFiles = [];
+			for (const f of files) {
+				const content = yield* ctx.world.readFile(f);
+				if (content && re.test(content)) matchedFiles.push(f);
+			}
+			return applyMode(matchedFiles.length > 0, ctx.mode, `code.pattern_exists: /${a.pattern}/ found in ${matchedFiles.slice(0, 3).join(", ")}`, `code.pattern_exists: /${a.pattern}/ not found in any ${a.glob}`);
+		});
+	}
+};
+//#endregion
+//#region src/runtime-checks/code-no-pattern.ts
+/**
+* Polarity-inverted alias of code.pattern_exists. "matched" means
+* "no pattern matches anywhere", which is the natural reading of the name.
+*/
+const codeNoPattern = {
+	type: "code.no_pattern",
+	verify(ctx) {
+		return Effect.gen(function* () {
+			const a = decodeCheckArgs(CodePatternArgsSchema, ctx.assertion, "code.no_pattern: assertion.glob and assertion.pattern must be non-empty strings");
+			if (isValidationFailure(a)) return a;
+			try {
+				new RegExp(a.pattern);
+			} catch (err) {
+				return {
+					pass: false,
+					score: 0,
+					reason: `code.no_pattern: invalid assertion.pattern regex: ${err instanceof Error ? err.message : String(err)}`
+				};
+			}
+			const inner = yield* codePatternExists.verify({
+				...ctx,
+				mode: "should"
+			});
+			return applyMode(!inner.pass, ctx.mode, `code.no_pattern: pattern absent`, `code.no_pattern: pattern present (${inner.reason})`);
+		});
+	}
+};
+//#endregion
+//#region src/runtime-checks/file-changes-outside-scope.ts
+const fileChangesOutsideScope = {
+	type: "file.changes_outside_scope",
+	verify(ctx) {
+		const a = decodeCheckArgs(FileChangesOutsideScopeArgsSchema, ctx.assertion, "file.changes_outside_scope: assertion.scope must contain at least one non-empty string");
+		if (isValidationFailure(a)) return Effect.succeed(a);
+		const unrelated = ctx.evidence.filesWritten().filter((f) => !a.scope.some((s) => f.path.startsWith(s)));
+		const matched = unrelated.length > 0;
+		return Effect.succeed(applyMode(matched, ctx.mode, `file.changes_outside_scope: ${unrelated.length} file(s) outside scope: ${unrelated.slice(0, 3).map((f) => f.path).join(", ")}`, `file.changes_outside_scope: no changes outside scope`));
+	}
+};
+//#endregion
+//#region src/runtime-checks/_match.ts
+function matchesSubset(actual, expected) {
+	if (expected === actual) return true;
+	if (Array.isArray(expected)) return Array.isArray(actual) && expected.length === actual.length && expected.every((item, i) => matchesSubset(actual[i], item));
+	if (expected && typeof expected === "object") {
+		if (!actual || typeof actual !== "object" || Array.isArray(actual)) return false;
+		const actualRecord = actual;
+		return Object.entries(expected).every(([key, value]) => matchesSubset(actualRecord[key], value));
+	}
+	return false;
+}
+//#endregion
+//#region src/runtime-checks/_call-match.ts
+function matchesRecordedCall(call, args) {
+	return (!args.tool || call.tool === args.tool) && (!args.provider || call.provider === args.provider) && (!args.server || call.server === args.server) && (args.args_match === void 0 || matchesSubset(call.args, args.args_match));
+}
+//#endregion
+//#region src/runtime-checks/tool-called.ts
+const toolCalled = {
+	type: "tool.called",
+	verify({ assertion, evidence, mode }) {
+		const a = decodeCheckArgs(ToolCalledArgsSchema, assertion, "tool.called: assertion.tool must be a non-empty string");
+		if (isValidationFailure(a)) return Effect.succeed(a);
+		const calls = evidence.toolCalls();
+		if (!calls.length) return Effect.succeed({
+			pass: false,
+			score: 0,
+			reason: "tool.called: no built-in tool evidence found"
+		});
+		const found = calls.some((c) => matchesRecordedCall(c, a));
+		return Effect.succeed(applyMode(found, mode, "tool.called: matched built-in tool call", "tool.called: matching call not found"));
+	}
+};
+//#endregion
+//#region src/runtime-checks/tool-not-called.ts
+const toolNotCalled = {
+	type: "tool.not_called",
+	verify({ assertion, evidence }) {
+		const a = decodeToolNotCalledArgs(assertion);
+		if (isValidationFailure(a)) return Effect.succeed(a);
+		const found = evidence.toolCalls().some((c) => matchesRecordedCall(c, a));
+		return Effect.succeed(found ? {
+			pass: false,
+			score: 0,
+			reason: "tool.not_called: forbidden built-in tool call observed"
+		} : {
+			pass: true,
+			score: 1,
+			reason: "tool.not_called: no matching built-in tool calls observed"
+		});
+	}
+};
+//#endregion
+//#region src/runtime-checks/skill-loaded.ts
+const skillLoaded = {
+	type: "skill.loaded",
+	verify({ assertion, evidence, mode }) {
+		const a = decodeCheckArgs(SkillLoadedArgsSchema, assertion, "skill.loaded: assertion must be an object");
+		if (isValidationFailure(a)) return Effect.succeed(a);
+		const shouldInclude = a.should_include ?? [];
+		const shouldExclude = a.should_exclude ?? [];
+		if (shouldInclude.length === 0 && shouldExclude.length === 0) return Effect.succeed({
+			pass: false,
+			score: 0,
+			reason: "skill.loaded: declare should_include or should_exclude"
+		});
+		const events = evidence.skillsLoaded().filter((event) => matchesLoad(event, a));
+		const loaded = new Set(events.map((event) => event.skill));
+		const missing = shouldInclude.filter((skill) => !loaded.has(skill));
+		const forbidden = shouldExclude.filter((skill) => loaded.has(skill));
+		const matched = missing.length === 0 && forbidden.length === 0;
+		const unmatchedReason = skillLoadedMismatchReason(missing, forbidden);
+		return Effect.succeed(applyMode(matched, mode, "skill.loaded: expected skill context observed", unmatchedReason));
+	}
+};
+function skillLoadedMismatchReason(missing, forbidden) {
+	return `skill.loaded: ${[missing.length ? `missing loaded skill(s): ${missing.join(", ")}` : "", forbidden.length ? `forbidden loaded skill(s): ${forbidden.join(", ")}` : ""].filter(Boolean).join("; ")}`;
+}
+function matchesLoad(event, args) {
+	return (!args.delivery || event.delivery === args.delivery) && (!args.provider || event.provider === args.provider) && (!args.server || event.server === args.server) && (!args.source || event.source === args.source);
+}
+//#endregion
+//#region src/runtime-checks/index.ts
+const corePlugins = [
+	verifierSucceeds,
+	verifierFails,
+	fileExists,
+	fileCreated,
+	fileContains,
+	fileNotModified,
+	fileChangesOutsideScope,
+	codePatternExists,
+	codeNoPattern,
+	toolCalled,
+	toolNotCalled,
+	skillLoaded
+];
+function buildRegistry(extra = []) {
+	const reg = /* @__PURE__ */ new Map();
+	for (const p of [...corePlugins, ...extra]) reg.set(p.type, p);
+	return reg;
+}
+buildRegistry();
+//#endregion
+//#region src/runtime-checks/catalog.ts
+var RuntimeCheckCatalog = class extends Context.Tag("agent-skill-evals/promptfoo/RuntimeCheckCatalog")() {};
+function runtimeCheckCatalogFromChecks(checks = corePlugins) {
+	const registry = /* @__PURE__ */ new Map();
+	for (const check of checks) registry.set(check.type, check);
+	return {
+		all: () => [...registry.values()],
+		get: (type) => registry.get(type),
+		knownTypes: () => new Set(registry.keys())
+	};
+}
+function makeRuntimeCheckCatalogLayer(extraChecks = []) {
+	return Layer.succeed(RuntimeCheckCatalog, runtimeCheckCatalogFromChecks([...corePlugins, ...extraChecks]));
+}
+const RuntimeCheckCatalogLive = makeRuntimeCheckCatalogLayer();
+function getRuntimeCheck(type) {
+	return Effect.map(RuntimeCheckCatalog, (catalog) => catalog.get(type));
+}
+Effect.map(RuntimeCheckCatalog, (catalog) => catalog.knownTypes());
+//#endregion
+//#region src/evidence-types.ts
+const EVIDENCE_SCHEMA_VERSION = "agent-skill-evals.evidence.v1";
+//#endregion
+//#region src/evidence-schema.ts
+const OptionalString = Schema.optional(Schema.String);
+const OptionalNumber = Schema.optional(Schema.Number);
+const StringArray = Schema.mutable(Schema.Array(Schema.String));
+const CommandEventSchema = Schema.Struct({
+	command: Schema.String,
+	args: Schema.optionalWith(StringArray, { default: () => [] }),
+	exitCode: Schema.Number,
+	signal: OptionalString,
+	stdout: OptionalString,
+	stderr: OptionalString,
+	startedAt: Schema.Number,
+	durationMs: Schema.Number
+});
+const FileEventSchema = Schema.Struct({
+	path: Schema.String,
+	op: Schema.Literal("create", "modify", "delete")
+});
+const ToolCallEventSchema = Schema.Struct({
+	tool: Schema.String,
+	provider: OptionalString,
+	server: OptionalString,
+	args: Schema.optional(Schema.Unknown),
+	result: Schema.optional(Schema.Unknown),
+	startedAt: Schema.Number,
+	durationMs: Schema.Number
+});
+const SkillLoadEventSchema = Schema.Struct({
+	skill: Schema.String,
+	delivery: Schema.Literal("native", "mcp"),
+	provider: OptionalString,
+	server: OptionalString,
+	source: OptionalString,
+	startedAt: Schema.Number
+});
+const UsageSchema = Schema.Struct({
+	inputTokens: OptionalNumber,
+	outputTokens: OptionalNumber,
+	totalTokens: OptionalNumber,
+	cacheReadTokens: OptionalNumber,
+	cacheWriteTokens: OptionalNumber
+});
+const RunSummarySchema = Schema.Struct({
+	runDir: Schema.String,
+	worldPath: Schema.String,
+	fixture: Schema.String,
+	durationMs: OptionalNumber
+});
+const EvidenceSnapshotSchema = Schema.Struct({
+	schemaVersion: Schema.Literal(EVIDENCE_SCHEMA_VERSION),
+	output: Schema.optionalWith(Schema.String, { default: () => "" }),
+	run: RunSummarySchema,
+	commands: Schema.optionalWith(Schema.mutable(Schema.Array(CommandEventSchema)), { default: () => [] }),
+	filesWritten: Schema.optionalWith(Schema.mutable(Schema.Array(FileEventSchema)), { default: () => [] }),
+	toolCalls: Schema.optionalWith(Schema.mutable(Schema.Array(ToolCallEventSchema)), { default: () => [] }),
+	skillsLoaded: Schema.optionalWith(Schema.mutable(Schema.Array(SkillLoadEventSchema)), { default: () => [] }),
+	usage: Schema.optionalWith(UsageSchema, { default: () => ({}) }),
+	extensions: Schema.optional(Schema.Record({
+		key: Schema.String,
+		value: Schema.Unknown
+	}))
+});
+const decodeEvidenceSnapshot = Schema.decodeUnknownEither(EvidenceSnapshotSchema, { errors: "all" });
+function decodeEvidenceSnapshotEither(input) {
+	const decoded = decodeEvidenceSnapshot(input);
+	if (Either.isRight(decoded)) return Either.right(decoded.right);
+	return Either.left(new Error(ParseResult.TreeFormatter.formatErrorSync(decoded.left)));
+}
+function parseEvidenceSnapshot(input) {
+	const decoded = decodeEvidenceSnapshotEither(input);
+	if (Either.isRight(decoded)) return decoded.right;
+	throw decoded.left;
+}
+//#endregion
+//#region src/agent/evidence.ts
+const DEFAULT_SKILL_EVIDENCE_CONFIG = {
+	mcpResource: {
+		uriArgPaths: ["uri"],
+		uriPatterns: ["^skill://(?<skill>[^/]+)/SKILL\\.md$"]
+	},
+	mcpTool: { toolPatterns: ["^load_(?<skill>[A-Za-z0-9_-]+)_skill$"] }
+};
+var EvidenceCollector = class EvidenceCollector {
+	skillEvidenceConfig;
+	snapshot = {
+		schemaVersion: EVIDENCE_SCHEMA_VERSION,
+		output: "",
+		run: {
+			runDir: "",
+			worldPath: "",
+			fixture: ""
+		},
+		commands: [],
+		filesWritten: [],
+		toolCalls: [],
+		skillsLoaded: [],
+		usage: {}
+	};
+	constructor(skillEvidenceConfig = {}) {
+		this.skillEvidenceConfig = mergeSkillEvidenceConfig(skillEvidenceConfig);
+	}
+	addCommand(e) {
+		this.snapshot.commands.push(e);
+	}
+	addFileWrite(e) {
+		this.snapshot.filesWritten.push(e);
+	}
+	addToolCall(e) {
+		this.snapshot.toolCalls.push(e);
+		const skillLoad = skillLoadFromToolCall(e, this.skillEvidenceConfig);
+		if (skillLoad) this.addSkillLoad(skillLoad);
+	}
+	addSkillLoad(e) {
+		if (this.snapshot.skillsLoaded.some((existing) => existing.skill === e.skill && existing.delivery === e.delivery && existing.provider === e.provider && existing.server === e.server)) return;
+		this.snapshot.skillsLoaded.push(e);
+	}
+	setUsage(u) {
+		this.snapshot.usage = u;
+	}
+	addUsage(u) {
+		this.snapshot.usage = mergeUsage(this.snapshot.usage, u);
+	}
+	setOutput(output) {
+		this.snapshot.output = output;
+	}
+	setRun(run) {
+		this.snapshot.run = run;
+	}
+	toSnapshot() {
+		return parseEvidenceSnapshot({
+			schemaVersion: this.snapshot.schemaVersion,
+			output: this.snapshot.output,
+			run: { ...this.snapshot.run },
+			commands: [...this.snapshot.commands],
+			filesWritten: [...this.snapshot.filesWritten],
+			toolCalls: [...this.snapshot.toolCalls],
+			skillsLoaded: [...this.snapshot.skillsLoaded],
+			usage: { ...this.snapshot.usage },
+			extensions: this.snapshot.extensions ? { ...this.snapshot.extensions } : void 0
+		});
+	}
+	async writeTo(runDir) {
+		return Effect.runPromise(writeEvidenceToEffect(this, runDir).pipe(Effect.provide(NodeServicesLive)));
+	}
+	static fromSnapshot(snapshot) {
+		const collector = new EvidenceCollector();
+		collector.snapshot = parseEvidenceSnapshot(snapshot);
+		return collector;
+	}
+};
+function mergeSkillEvidenceConfig(config) {
+	return {
+		mcpResource: {
+			uriArgPaths: config.mcpResource?.uriArgPaths ?? DEFAULT_SKILL_EVIDENCE_CONFIG.mcpResource.uriArgPaths,
+			uriPatterns: config.mcpResource?.uriPatterns ?? DEFAULT_SKILL_EVIDENCE_CONFIG.mcpResource.uriPatterns
+		},
+		mcpTool: { toolPatterns: config.mcpTool?.toolPatterns ?? DEFAULT_SKILL_EVIDENCE_CONFIG.mcpTool.toolPatterns },
+		...config.nativeArgs ? { nativeArgs: config.nativeArgs } : {}
+	};
+}
+function skillLoadFromToolCall(event, config) {
+	const uri = skillUriFromArgs(event.args, config.mcpResource?.uriArgPaths ?? []);
+	const skill = uri ? skillFromPattern(uri, config.mcpResource?.uriPatterns ?? []) : skillFromTool(event.tool, config.mcpTool?.toolPatterns ?? []);
+	if (!skill) return void 0;
+	const server = event.server ?? serverFromArgs(event.args);
+	return {
+		skill,
+		delivery: "mcp",
+		...event.provider ? { provider: event.provider } : {},
+		...server ? { server } : {},
+		source: event.tool,
+		startedAt: event.startedAt
+	};
+}
+function skillUriFromArgs(args, paths) {
+	for (const path of paths) {
+		const uri = valueAtPath(args, path);
+		if (typeof uri === "string") return uri;
+	}
+}
+function serverFromArgs(args) {
+	if (!args || typeof args !== "object" || Array.isArray(args)) return void 0;
+	const server = args.server;
+	return typeof server === "string" ? server : void 0;
+}
+function skillFromPattern(value, patterns) {
+	for (const pattern of patterns) {
+		let regex;
+		try {
+			regex = new RegExp(pattern, "i");
+		} catch {
+			continue;
+		}
+		const match = regex.exec(value);
+		const skill = match?.groups?.skill ?? match?.[1];
+		if (skill) return skill;
+	}
+}
+function skillFromTool(tool, patterns) {
+	return skillFromPattern(tool, patterns)?.replace(/_/g, "-");
+}
+function valueAtPath(value, path) {
+	let current = value;
+	for (const segment of path.split(".")) {
+		if (!current || typeof current !== "object" || Array.isArray(current)) return void 0;
+		current = current[segment];
+	}
+	return current;
+}
+function addOptionalNumbers(a, b) {
+	if (a === void 0) return b;
+	if (b === void 0) return a;
+	return a + b;
+}
+function mergeUsage(a, b) {
+	return {
+		inputTokens: addOptionalNumbers(a.inputTokens, b.inputTokens),
+		outputTokens: addOptionalNumbers(a.outputTokens, b.outputTokens),
+		totalTokens: addOptionalNumbers(a.totalTokens, b.totalTokens),
+		cacheReadTokens: addOptionalNumbers(a.cacheReadTokens, b.cacheReadTokens),
+		cacheWriteTokens: addOptionalNumbers(a.cacheWriteTokens, b.cacheWriteTokens)
+	};
+}
+function writeEvidenceToEffect(collector, runDir) {
+	return Effect.gen(function* () {
+		const fs = yield* FileSystem;
+		const path = join(runDir, "evidence.json");
+		yield* fs.writeText(path, JSON.stringify(collector.toSnapshot(), null, 2));
+		return path;
+	});
+}
+function evidenceFromSnapshot(s) {
+	return {
+		commands: () => s.commands,
+		filesWritten: () => s.filesWritten,
+		toolCalls: () => s.toolCalls,
+		skillsLoaded: () => s.skillsLoaded,
+		usage: () => s.usage
+	};
+}
+//#endregion
+//#region src/agent/command-runner.ts
+const DEFAULT_OUTPUT_LIMIT = 4096;
+const KILL_GRACE_MS = 1e3;
+const EXIT_STDIO_FLUSH_MS = 50;
+var ProcessRunner = class extends Context.Tag("agent-skill-evals/promptfoo/ProcessRunner")() {};
+function appendLimited(current, chunk, limit) {
+	if (limit <= 0) return "";
+	const next = current + chunk;
+	return next.length <= limit ? next : next.slice(next.length - limit);
+}
+function killProcessGroup(child, signal) {
+	if (!child.pid) return;
+	try {
+		if (process.platform === "win32") child.kill(signal);
+		else process.kill(-child.pid, signal);
+	} catch {
+		try {
+			child.kill(signal);
+		} catch {}
+	}
+}
+function runCommandEffect(command, args = [], options) {
+	return Effect.flatMap(ProcessRunner, (runner) => runner.run(command, args, options));
+}
+const ProcessRunnerLive = Layer.succeed(ProcessRunner, { run: nodeRunCommandEffect });
+function nodeRunCommandEffect(command, args, options) {
+	const stdoutLimit = options.stdoutLimit ?? DEFAULT_OUTPUT_LIMIT;
+	const stderrLimit = options.stderrLimit ?? DEFAULT_OUTPUT_LIMIT;
+	return Effect.gen(function* () {
+		const startedAt = yield* Clock.currentTimeMillis;
+		return yield* Effect.async((resume, signal) => {
+			const child = spawn(command, [...args], {
+				cwd: options.cwd,
+				env: options.env,
+				detached: process.platform !== "win32",
+				stdio: [
+					"pipe",
+					"pipe",
+					"pipe"
+				]
+			});
+			let stdout = "";
+			let stderr = "";
+			let timedOut = false;
+			let settled = false;
+			let timeoutTimer = null;
+			let forceTimer = null;
+			let exitFlushTimer = null;
+			let abortListener = null;
+			const finish = (result) => {
+				if (settled) return;
+				settled = true;
+				if (timeoutTimer) clearTimeout(timeoutTimer);
+				if (forceTimer) clearTimeout(forceTimer);
+				if (exitFlushTimer) clearTimeout(exitFlushTimer);
+				child.stdout?.destroy();
+				child.stderr?.destroy();
+				child.stdin?.destroy();
+				if (abortListener) signal.removeEventListener("abort", abortListener);
+				if (process.platform !== "win32") killProcessGroup(child, "SIGTERM");
+				resume(Clock.currentTimeMillis.pipe(Effect.map((endedAt) => ({
+					...result,
+					durationMs: endedAt - startedAt
+				}))));
+			};
+			abortListener = () => {
+				timedOut = true;
+				stderr = appendLimited(stderr, `${stderr ? "\n" : ""}agent-skill-evals: command interrupted`, stderrLimit);
+				killProcessGroup(child, "SIGKILL");
+				finish({
+					command,
+					args: [...args],
+					exitCode: -1,
+					signal: "SIGKILL",
+					stdout,
+					stderr,
+					startedAt,
+					timedOut
+				});
+			};
+			signal.addEventListener("abort", abortListener, { once: true });
+			child.stdout?.on("data", (chunk) => {
+				const text = chunk.toString();
+				stdout = appendLimited(stdout, text, stdoutLimit);
+				options.onStdout?.(text);
+			});
+			child.stderr?.on("data", (chunk) => {
+				const text = chunk.toString();
+				stderr = appendLimited(stderr, text, stderrLimit);
+				options.onStderr?.(text);
+			});
+			child.on("error", (error) => {
+				stderr = appendLimited(stderr, String(error), stderrLimit);
+				finish({
+					command,
+					args: [...args],
+					exitCode: -1,
+					stdout,
+					stderr,
+					startedAt,
+					timedOut,
+					error
+				});
+			});
+			child.on("exit", (code, signal) => {
+				const result = {
+					command,
+					args: [...args],
+					exitCode: code ?? -1,
+					signal: signal ?? void 0,
+					stdout,
+					stderr,
+					startedAt,
+					timedOut
+				};
+				exitFlushTimer = setTimeout(() => finish(result), EXIT_STDIO_FLUSH_MS);
+			});
+			child.on("close", (code, signal) => {
+				finish({
+					command,
+					args: [...args],
+					exitCode: code ?? -1,
+					signal: signal ?? void 0,
+					stdout,
+					stderr,
+					startedAt,
+					timedOut
+				});
+			});
+			if (options.stdin !== void 0) child.stdin?.write(options.stdin);
+			child.stdin?.end();
+			if (options.timeoutMs) timeoutTimer = setTimeout(() => {
+				timedOut = true;
+				stderr = appendLimited(stderr, `${stderr ? "\n" : ""}agent-skill-evals: command timed out after ${options.timeoutMs}ms`, stderrLimit);
+				killProcessGroup(child, "SIGKILL");
+				forceTimer = setTimeout(() => {
+					finish({
+						command,
+						args: [...args],
+						exitCode: -1,
+						signal: "SIGKILL",
+						stdout,
+						stderr,
+						startedAt,
+						timedOut
+					});
+				}, KILL_GRACE_MS);
+			}, options.timeoutMs);
+		});
+	});
+}
+//#endregion
+//#region src/agent/jsonl-stream.ts
+function appendLine(events, line) {
+	if (!line.trim()) return events;
+	try {
+		return [...events, JSON.parse(line)];
+	} catch {
+		return events;
+	}
+}
+function parseChunk(state, text) {
+	const lines = (state.leftover + text).split("\n");
+	return {
+		leftover: lines.pop() ?? "",
+		events: lines.reduce(appendLine, state.events)
+	};
+}
+function createJsonlEventParser() {
+	let leftover = "";
+	return {
+		push(chunk) {
+			const state = parseChunk({
+				leftover,
+				events: []
+			}, chunk);
+			leftover = state.leftover;
+			return state.events;
+		},
+		finish() {
+			const events = appendLine([], leftover);
+			leftover = "";
+			return events;
+		}
+	};
+}
+//#endregion
+//#region src/agent/adapters.ts
+function argsWithPrompt(args, prompt) {
+	const dashIndex = args.lastIndexOf("-");
+	if (dashIndex === -1) return [...args, prompt];
+	return args.map((arg, index) => index === dashIndex ? prompt : arg);
+}
+function normalizePathFromCwd(path, cwd) {
+	if (!isAbsolute(path)) return path;
+	const candidate = path.startsWith("/private/") ? path.slice(8) : path;
+	const rel = relative(cwd.startsWith("/private/") ? cwd.slice(8) : cwd, candidate);
+	return rel && !rel.startsWith("..") && !isAbsolute(rel) ? rel : path;
+}
+function normalizeToolCallArgs(args, cwd) {
+	if (!args || typeof args !== "object" || Array.isArray(args)) return args;
+	const normalized = {};
+	for (const [key, value] of Object.entries(args)) {
+		const normalizedValue = (key === "path" || key === "file_path") && typeof value === "string" ? normalizePathFromCwd(value, cwd) : value;
+		normalized[key] = normalizedValue;
+		if (key === "file_path" && typeof normalizedValue === "string" && normalized.path === void 0) normalized.path = normalizedValue;
+	}
+	return normalized;
+}
+function normalizeMcpToolCall(event) {
+	const match = /^mcp__(.+?)__(.+)$/.exec(event.tool);
+	if (!match) return event;
+	return {
+		...event,
+		server: event.server ?? match[1],
+		tool: match[2] ?? event.tool
+	};
+}
+function parseJsonObjectString(value) {
+	if (typeof value !== "string") return value;
+	try {
+		const parsed = JSON.parse(value);
+		return parsed && typeof parsed === "object" ? parsed : value;
+	} catch {
+		return value;
+	}
+}
+function evidenceWithRelativeToolPaths(evidence, cwd, now) {
+	return {
+		addCommand: (event) => evidence.addCommand(event),
+		addToolCall: (event) => {
+			const normalized = normalizeMcpToolCall(event);
+			evidence.addToolCall({
+				...normalized,
+				args: normalizeToolCallArgs(normalized.args, cwd)
+			});
+		},
+		setUsage: (usage) => evidence.setUsage(usage),
+		addUsage: (usage) => evidence.addUsage(usage),
+		now
+	};
+}
+function normalizeUsage(usage) {
+	const inputTokens = usage.input_tokens ?? usage.inputTokens ?? usage.input;
+	const outputTokens = usage.output_tokens ?? usage.outputTokens ?? usage.output;
+	const cacheReadTokens = usage.cache_read_input_tokens ?? usage.cacheReadTokens ?? usage.cacheRead;
+	const cacheWriteTokens = usage.cache_creation_input_tokens ?? usage.cacheWriteTokens ?? usage.cacheWrite;
+	return {
+		inputTokens,
+		outputTokens,
+		totalTokens: usage.total_tokens ?? usage.totalTokens ?? addKnownNumbers(inputTokens, outputTokens),
+		cacheReadTokens,
+		cacheWriteTokens
+	};
+}
+function addKnownNumbers(...values) {
+	const present = values.filter((value) => typeof value === "number");
+	return present.length > 0 ? present.reduce((sum, value) => sum + value, 0) : void 0;
+}
+function normalizePiToolName(tool) {
+	switch (tool.toLowerCase()) {
+		case "bash": return "Bash";
+		case "edit": return "Edit";
+		case "read": return "Read";
+		case "write": return "Write";
+		default: return tool;
+	}
+}
+function evidenceStartedAt(evidence) {
+	return evidence.now?.() ?? 0;
+}
+function runJsonlAdapter(input, onEvent, options = {}) {
+	const { command, args, cwd, prompt, evidence, timeoutMs, env } = input;
+	const promptDelivery = options.promptDelivery ?? "stdin";
+	const spawnArgs = promptDelivery === "arg" ? argsWithPrompt(args, prompt) : [...args];
+	return Effect.gen(function* () {
+		const runStartedAt = yield* Clock.currentTimeMillis;
+		const adapterEvidence = evidenceWithRelativeToolPaths(evidence, cwd, () => runStartedAt);
+		const parser = createJsonlEventParser();
+		let finalText = "";
+		const handleChunk = (chunk) => {
+			for (const evt of parser.push(chunk)) onEvent(evt, adapterEvidence, (text) => finalText += text);
+		};
+		const result = yield* runCommandEffect(command, spawnArgs, {
+			cwd,
+			env,
+			stdin: promptDelivery === "stdin" ? prompt : void 0,
+			timeoutMs,
+			stdoutLimit: 0,
+			onStdout: handleChunk
+		});
+		for (const evt of parser.finish()) onEvent(evt, adapterEvidence, (text) => finalText += text);
+		evidence.addCommand({
+			command,
+			args: [...spawnArgs],
+			exitCode: result.exitCode,
+			stderr: result.stderr.slice(0, 4096),
+			startedAt: result.startedAt,
+			durationMs: result.durationMs
+		});
+		const error = result.error ? `adapter error: failed to start "${command}": ${result.error.message}` : result.timedOut ? `${command} timed out after ${timeoutMs ?? 0}ms` : result.exitCode !== 0 ? `${command} exited ${result.exitCode}${result.stderr.trim() ? `: ${result.stderr.trim()}` : ""}` : void 0;
+		if (error && !finalText.trim()) finalText = error;
+		return {
+			output: finalText.trim(),
+			exitCode: result.exitCode,
+			timedOut: result.timedOut,
+			...error ? { error } : {},
+			durationMs: result.durationMs
+		};
+	});
+}
+/**
+* Claude Code stream-json adapter: parses stream-json events emitted by
+* `claude -p ... --output-format stream-json` and projects them into evidence.
+*
+* Events of interest (see Claude Code docs):
+*  - { type: "system" | "assistant" | "user" | "result", ... }
+*  - tool_use blocks inside assistant content (Bash, Edit, Write, etc.)
+*/
+const claudeCodeJsonAdapter = {
+	id: "claude-code-json",
+	run(input) {
+		return runJsonlAdapter(input, handleClaudeEvent);
+	}
+};
+const codexJsonAdapter = {
+	id: "codex-json",
+	run(input) {
+		return runJsonlAdapter(input, handleCodexEvent, { promptDelivery: "arg" });
+	}
+};
+const piJsonAdapter = {
+	id: "pi-json",
+	run(input) {
+		return runJsonlAdapter(input, handlePiEvent);
+	}
+};
+const internalTestJsonAdapter = {
+	id: "internal-test-json",
+	run(input) {
+		return runJsonlAdapter(input, handleCodexEvent);
+	}
+};
+function handleClaudeEvent(evt, evidence, appendFinal) {
+	if (!evt || typeof evt !== "object") return;
+	const e = evt;
+	if (e.type === "result" && typeof e.result === "string") appendFinal(e.result);
+	if (e.type === "assistant" && e.message && typeof e.message === "object") {
+		const msg = e.message;
+		if (msg.usage && typeof msg.usage === "object") evidence.addUsage(normalizeUsage(msg.usage));
+		if (Array.isArray(msg.content)) for (const block of msg.content) {
+			if (!block || typeof block !== "object") continue;
+			const b = block;
+			if (b.type === "text" && typeof b.text === "string") appendFinal(b.text);
+			if (b.type === "tool_use" && typeof b.name === "string") evidence.addToolCall({
+				tool: b.name,
+				provider: "claude-code-json",
+				args: b.input,
+				startedAt: evidenceStartedAt(evidence),
+				durationMs: 0
+			});
+		}
+	}
+	if (e.type === "result" && e.usage && typeof e.usage === "object") evidence.setUsage(normalizeUsage(e.usage));
+}
+function handleCodexEvent(evt, evidence, appendFinal) {
+	if (!evt || typeof evt !== "object") return;
+	const e = evt;
+	const type = typeof e.type === "string" ? e.type : "";
+	const text = typeof e.message === "string" ? e.message : typeof e.text === "string" ? e.text : typeof e.content === "string" ? e.content : typeof e.output === "string" ? e.output : void 0;
+	if (text && /message|final|result|response|output/.test(type)) appendFinal(text);
+	const item = e.item && typeof e.item === "object" ? e.item : e;
+	const itemType = typeof item.type === "string" ? item.type : "";
+	const toolName = typeof item.tool === "string" ? item.tool : typeof item.name === "string" && /tool|call|command/.test(`${type}:${itemType}`) ? item.name : void 0;
+	if (toolName && /tool|call|command/.test(`${type}:${itemType}`)) evidence.addToolCall({
+		tool: toolName,
+		provider: "codex-json",
+		server: typeof item.server === "string" ? item.server : void 0,
+		args: parseJsonObjectString(item.args ?? item.input ?? item.arguments),
+		result: item.result ?? item.output,
+		startedAt: evidenceStartedAt(evidence),
+		durationMs: 0
+	});
+	if (itemType === "file_change" && Array.isArray(item.changes)) for (const change of item.changes) {
+		if (!change || typeof change !== "object") continue;
+		const c = change;
+		if (typeof c.path !== "string") continue;
+		evidence.addToolCall({
+			tool: "Edit",
+			provider: "codex-json",
+			args: {
+				path: c.path,
+				kind: typeof c.kind === "string" ? c.kind : void 0
+			},
+			startedAt: evidenceStartedAt(evidence),
+			durationMs: 0
+		});
+	}
+	if ((type.includes("exec") || type.includes("command") || itemType.includes("command")) && typeof item.command === "string") {
+		const args = Array.isArray(item.args) ? item.args.map(String) : [];
+		evidence.addCommand({
+			command: item.command,
+			args,
+			exitCode: typeof item.exit_code === "number" ? item.exit_code : typeof item.exitCode === "number" ? item.exitCode : 0,
+			stdout: typeof item.stdout === "string" ? item.stdout.slice(0, 4096) : typeof item.aggregated_output === "string" ? item.aggregated_output.slice(0, 4096) : void 0,
+			stderr: typeof item.stderr === "string" ? item.stderr.slice(0, 4096) : void 0,
+			startedAt: evidenceStartedAt(evidence),
+			durationMs: typeof item.durationMs === "number" ? item.durationMs : 0
+		});
+	}
+	const usage = e.usage && typeof e.usage === "object" ? e.usage : void 0;
+	if (usage) evidence.setUsage(normalizeUsage(usage));
+}
+function handlePiEvent(evt, evidence, appendFinal) {
+	if (!evt || typeof evt !== "object") return;
+	const e = evt;
+	const type = typeof e.type === "string" ? e.type : "";
+	const message = e.message && typeof e.message === "object" ? e.message : void 0;
+	const messageUsage = message?.usage && typeof message.usage === "object" ? message.usage : void 0;
+	if (messageUsage && (type === "message_end" || type === "turn_end" || type === "agent_end")) evidence.setUsage(normalizeUsage(messageUsage));
+	const text = typeof e.message === "string" ? e.message : typeof e.text === "string" ? e.text : typeof e.content === "string" ? e.content : typeof e.output === "string" ? e.output : typeof e.result === "string" ? e.result : void 0;
+	if (text && /assistant|message|final|result|response|output/.test(type)) appendFinal(text);
+	if (type === "tool_execution_start" || type === "tool_execution_end") {
+		const rawTool = typeof e.tool === "string" ? e.tool : typeof e.name === "string" ? e.name : typeof e.tool_name === "string" ? e.tool_name : typeof e.toolName === "string" ? e.toolName : void 0;
+		const tool = rawTool ? normalizePiToolName(rawTool) : void 0;
+		if (tool) evidence.addToolCall({
+			tool,
+			provider: "pi-json",
+			args: e.args ?? e.input ?? e.arguments,
+			result: type === "tool_execution_end" ? e.result ?? e.output : void 0,
+			startedAt: evidenceStartedAt(evidence),
+			durationMs: typeof e.duration_ms === "number" ? e.duration_ms : typeof e.durationMs === "number" ? e.durationMs : 0
+		});
+	}
+	if (type === "tool_execution_end") {
+		const tool = typeof e.tool === "string" ? e.tool : typeof e.name === "string" ? e.name : typeof e.tool_name === "string" ? e.tool_name : typeof e.toolName === "string" ? e.toolName : "";
+		const commandText = typeof e.command === "string" ? e.command : e.args && typeof e.args === "object" && !Array.isArray(e.args) && typeof e.args.command === "string" ? e.args.command : void 0;
+		if (commandText && /bash|shell|command|exec/i.test(tool)) evidence.addCommand({
+			command: commandText,
+			args: [],
+			exitCode: typeof e.exit_code === "number" ? e.exit_code : typeof e.exitCode === "number" ? e.exitCode : 0,
+			stdout: typeof e.stdout === "string" ? e.stdout.slice(0, 4096) : typeof e.output === "string" ? e.output.slice(0, 4096) : void 0,
+			stderr: typeof e.stderr === "string" ? e.stderr.slice(0, 4096) : void 0,
+			startedAt: evidenceStartedAt(evidence),
+			durationMs: typeof e.duration_ms === "number" ? e.duration_ms : typeof e.durationMs === "number" ? e.durationMs : 0
+		});
+	}
+	const usage = e.usage && typeof e.usage === "object" ? e.usage : e;
+	if (type === "usage" || e.usage) evidence.setUsage(normalizeUsage(usage));
+}
+const adapterRegistry = new Map([
+	[claudeCodeJsonAdapter.id, claudeCodeJsonAdapter],
+	[codexJsonAdapter.id, codexJsonAdapter],
+	[piJsonAdapter.id, piJsonAdapter],
+	[internalTestJsonAdapter.id, internalTestJsonAdapter]
+]);
+//#endregion
+//#region src/agent/adapter-catalog.ts
+var AdapterCatalog = class extends Context.Tag("agent-skill-evals/promptfoo/AdapterCatalog")() {};
+const AdapterCatalogLive = Layer.succeed(AdapterCatalog, { get: (id) => adapterRegistry.get(id) });
+function getAdapter(id) {
+	return Effect.map(AdapterCatalog, (catalog) => catalog.get(id));
+}
+//#endregion
+//#region src/runtime-checks/_files.ts
+const SKIP_DIRS = new Set(["node_modules", ".git"]);
+function walkFilesEffect(root, visit) {
+	return Effect.gen(function* () {
+		const fs = yield* FileSystem;
+		function walk(dir) {
+			return Effect.gen(function* () {
+				const entries = yield* fs.readDirectory(dir).pipe(Effect.catchAll(() => Effect.succeed([])));
+				for (const entry of entries) {
+					const absolutePath = join(dir, entry.name);
+					if (entry.isDirectory()) {
+						if (SKIP_DIRS.has(entry.name)) continue;
+						yield* walk(absolutePath);
+						continue;
+					}
+					if (entry.isFile()) yield* visit(relative(root, absolutePath), absolutePath);
+				}
+			});
+		}
+		if (!(yield* fs.stat(root).pipe(Effect.catchAll(() => Effect.succeed(null))))?.isDirectory()) return;
+		yield* walk(root);
+	});
+}
+function listMatchingFilesEffect(root, glob) {
+	return Effect.gen(function* () {
+		const matches = [];
+		yield* walkFilesEffect(root, (relativePath) => Effect.sync(() => {
+			if (matchesGlob(relativePath, glob)) matches.push(relativePath);
+		}));
+		return matches;
+	});
+}
+function matchesGlob(relativePath, glob) {
+	if (!glob.includes("/") && relativePath.includes("/")) return globToRegExp(glob).test(relativePath.split("/").at(-1) ?? relativePath);
+	return globToRegExp(glob).test(relativePath);
+}
+function globToRegExp(glob) {
+	const globstar = "__AGENT_SKILL_EVALS_GLOBSTAR__";
+	const star = "__AGENT_SKILL_EVALS_STAR__";
+	const pattern = glob.replace(/^\.\//, "").replace(/\*\*\//g, globstar).replace(/\*\*/g, globstar).replace(/\*/g, star).replace(/[.+^${}()|[\]\\]/g, "\\$&").replaceAll(globstar, "(?:.*/)?").replaceAll(star, "[^/]*");
+	return new RegExp(`^${pattern}$`);
+}
+//#endregion
+//#region src/agent/world.ts
+function createRunDirEffect() {
+	return Effect.gen(function* () {
+		const fs = yield* FileSystem;
+		const runDir = yield* fs.makeTempDirectory("agent-skill-evals-run-");
+		const worldPath = join(runDir, "world");
+		yield* fs.makeDirectory(worldPath);
+		return {
+			runDir,
+			worldPath
+		};
+	});
+}
+function copyFixtureEffect(input, worldPath) {
+	return Effect.gen(function* () {
+		const fs = yield* FileSystem;
+		const cwd = yield* (yield* Environment).cwd;
+		const src = isAbsolute(input.fixturePath) ? input.fixturePath : resolve(input.testFileDir ?? input.baseDir ?? cwd, input.fixturePath);
+		yield* fs.copyDirectory(src, worldPath);
+	});
+}
+function makeWorldHandle(worldPath, recordCommand) {
+	return {
+		path: worldPath,
+		readFile(rel) {
+			return Effect.gen(function* () {
+				return yield* (yield* FileSystem).readText(join(worldPath, rel)).pipe(Effect.catchAll(() => Effect.succeed(null)));
+			}).pipe(Effect.provide(NodeServicesLive));
+		},
+		listFiles(glob) {
+			return listMatchingFilesEffect(worldPath, glob).pipe(Effect.provide(NodeServicesLive));
+		},
+		exec(command, args = [], opts = {}) {
+			return Effect.gen(function* () {
+				const result = yield* runCommandEffect(command, args, {
+					cwd: worldPath,
+					env: {
+						...yield* (yield* Environment).env,
+						...opts.env ?? {}
+					},
+					timeoutMs: opts.timeoutMs
+				});
+				recordCommand?.({
+					command,
+					args: [...args],
+					exitCode: result.exitCode,
+					stdout: result.stdout.slice(0, 4096),
+					stderr: result.stderr.slice(0, 4096),
+					startedAt: result.startedAt,
+					durationMs: result.durationMs
+				});
+				return {
+					exitCode: result.exitCode,
+					stdout: result.stdout,
+					stderr: result.stderr
+				};
+			}).pipe(Effect.provide(ProcessRunnerLive), Effect.provide(NodeServicesLive));
+		},
+		diff() {
+			return Effect.succeed("");
+		}
+	};
+}
+//#endregion
+//#region src/agent/file-watch.ts
+function snapshotTreeEffect(root) {
+	return Effect.gen(function* () {
+		const out = /* @__PURE__ */ new Map();
+		const fs = yield* FileSystem;
+		function walk(dir) {
+			return Effect.gen(function* () {
+				const entries = yield* fs.readDirectory(dir).pipe(Effect.catchAll(() => Effect.succeed([])));
+				for (const e of entries) {
+					const p = join(dir, e.name);
+					if (e.isDirectory()) {
+						if (e.name === "node_modules" || e.name === ".git") continue;
+						yield* walk(p);
+					} else if (e.isFile()) {
+						const buf = yield* fs.readFile(p).pipe(Effect.catchAll(() => Effect.succeed(null)));
+						if (!buf) continue;
+						const hash = createHash("sha256").update(buf).digest("hex");
+						out.set(relative(root, p), hash);
+					}
+				}
+			});
+		}
+		yield* fs.stat(root).pipe(Effect.catchAll(() => Effect.succeed(null)));
+		yield* walk(root);
+		return out;
+	});
+}
+function diffTrees(before, after) {
+	const events = [];
+	for (const [path, hash] of after) {
+		const prev = before.get(path);
+		if (prev === void 0) events.push({
+			path,
+			op: "create"
+		});
+		else if (prev !== hash) events.push({
+			path,
+			op: "modify"
+		});
+	}
+	for (const [path] of before) if (!after.has(path)) events.push({
+		path,
+		op: "delete"
+	});
+	return events;
+}
+//#endregion
+//#region src/agent/index.ts
+const SkillEvidenceConfigSchema = Schema.Struct({
+	mcpResource: Schema.optional(Schema.Struct({
+		uriArgPaths: Schema.optional(Schema.Array(Schema.String)),
+		uriPatterns: Schema.optional(Schema.Array(Schema.String))
+	})),
+	mcpTool: Schema.optional(Schema.Struct({ toolPatterns: Schema.optional(Schema.Array(Schema.String)) })),
+	nativeArgs: Schema.optional(Schema.Struct({
+		whenArgs: Schema.optional(Schema.Array(Schema.String)),
+		whenAnyArgs: Schema.optional(Schema.Array(Schema.String)),
+		skillPathFlags: Schema.optional(Schema.Array(Schema.String)),
+		provider: Schema.optional(Schema.String),
+		source: Schema.optional(Schema.String)
+	}))
+});
+const ProviderConfigSchema = Schema.Struct({
+	adapter: Schema.optional(Schema.String),
+	command: Schema.optional(Schema.String),
+	args: Schema.optional(Schema.Array(Schema.String)),
+	timeoutMs: Schema.optional(Schema.Number),
+	baseDir: Schema.optional(Schema.String),
+	isolatedHome: Schema.optional(Schema.Boolean),
+	skillEvidence: Schema.optional(SkillEvidenceConfigSchema)
+});
+const DOCUMENTED_ADAPTERS = [
+	"codex-json",
+	"claude-code-json",
+	"pi-json"
+];
+const AgentLiveLayer = Layer.mergeAll(RuntimeCheckCatalogLive, AdapterCatalogLive, NodeServicesLive, ProcessRunnerLive);
+function asVars(value) {
+	return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function varsFromContext(context) {
+	return asVars(context.vars) ?? asVars(context.test?.vars) ?? {};
+}
+function decodeProviderConfig(input) {
+	const decoded = Schema.decodeUnknownEither(ProviderConfigSchema, { errors: "all" })(input ?? {});
+	if (Either.isRight(decoded)) return decoded.right;
+	return { error: `agent-skill-evals-provider: invalid config: ${ParseResult.TreeFormatter.formatErrorSync(decoded.left)}` };
+}
+function prepareRunEffect(fixture, config) {
+	return Effect.gen(function* () {
+		const created = yield* Effect.either(createRunDirEffect());
+		if (Either.isLeft(created)) return { error: `agent-skill-evals-provider: failed to create isolated world: ${created.left instanceof Error ? created.left.message : String(created.left)}` };
+		const { runDir, worldPath } = created.right;
+		const copied = yield* Effect.either(copyFixtureEffect({
+			fixturePath: fixture,
+			baseDir: config.baseDir
+		}, worldPath));
+		if (Either.isLeft(copied)) return { error: `agent-skill-evals-provider: failed to copy vars.fixture "${fixture}" into isolated world: ${copied.left instanceof Error ? copied.left.message : String(copied.left)}` };
+		const evidenceCollector = new EvidenceCollector(config.skillEvidence);
+		return {
+			runDir,
+			worldPath,
+			evidenceCollector,
+			world: makeWorldHandle(worldPath, (event) => evidenceCollector.addCommand(event))
+		};
+	});
+}
+function runPreconditionsEffect(vars, run) {
+	return Effect.gen(function* () {
+		const results = [];
+		let passed = true;
+		const parsed = parseRuntimeTestFields(vars);
+		for (const error of parsed.errors.filter((e) => e.field === "preconditions")) {
+			const at = error.index === void 0 ? error.field : `${error.field}[${error.index}]`;
+			results.push({
+				pass: false,
+				score: 0,
+				reason: `precondition: ${at}: ${error.reason}`
+			});
+			passed = false;
+		}
+		for (const entry of parsed.preconditions) {
+			const plugin = yield* getRuntimeCheck(entry.type);
+			if (!plugin) {
+				results.push({
+					pass: false,
+					score: 0,
+					reason: `precondition: unknown effect type "${entry.type}"`
+				});
+				passed = false;
+				continue;
+			}
+			const result = yield* plugin.verify({
+				assertion: entry.args,
+				world: run.world,
+				evidence: evidenceFromSnapshot(run.evidenceCollector.toSnapshot()),
+				mode: "precondition"
+			});
+			results.push(result);
+			if (!result.pass) passed = false;
+		}
+		return {
+			results,
+			passed
+		};
+	});
+}
+function resolveConfiguredPath(baseDir, path) {
+	if (path.includes("=")) return path;
+	return path.startsWith("./") || path.startsWith("../") || !isAbsolute(path) && path.includes("/") ? resolve(baseDir, path) : path;
+}
+function expandEnvVars(value, env) {
+	return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)(:-([^}]*))?\}/g, (_match, name, _fallbackPart, fallback) => env[name] ?? fallback ?? "");
+}
+function skillNameFromNativePath(path) {
+	const normalized = path.replace(/\/+$/, "");
+	if (!normalized) return void 0;
+	const leaf = basename(normalized);
+	return leaf === "SKILL.md" ? basename(dirname(normalized)) : leaf.replace(/\.md$/i, "");
+}
+function addNativeSkillEvidenceFromConfig(run, config, startedAt) {
+	const nativeConfig = nativeSkillEvidenceConfig(config);
+	if (!nativeConfig) return;
+	const args = config.args ?? [];
+	if (!nativeConfig.whenArgs.every((arg) => args.includes(arg))) return;
+	if (nativeConfig.whenAnyArgs.length > 0 && !nativeConfig.whenAnyArgs.some((arg) => args.includes(arg))) return;
+	for (let index = 0; index < args.length; index += 1) {
+		const arg = args[index];
+		if (!arg) continue;
+		const source = nativeConfig.skillPathFlags.includes(arg) ? arg : void 0;
+		if (!source) continue;
+		const skillPath = args[index + 1];
+		if (!skillPath) continue;
+		const skill = skillNameFromNativePath(skillPath);
+		if (!skill) continue;
+		run.evidenceCollector.addSkillLoad({
+			skill,
+			delivery: "native",
+			provider: nativeConfig.provider,
+			source: nativeConfig.source ?? source,
+			startedAt
+		});
+	}
+}
+function nativeSkillEvidenceConfig(config) {
+	const configured = config.skillEvidence?.nativeArgs;
+	if (configured) return {
+		whenArgs: configured.whenArgs ?? [],
+		whenAnyArgs: configured.whenAnyArgs ?? [],
+		skillPathFlags: configured.skillPathFlags ?? ["--skill"],
+		provider: configured.provider ?? config.adapter ?? "agent",
+		source: configured.source
+	};
+	if (config.adapter === "pi-json") return {
+		whenArgs: [],
+		whenAnyArgs: ["--no-skills", "-ns"],
+		skillPathFlags: ["--skill"],
+		provider: "pi-json",
+		source: "--skill"
+	};
+}
+function promptfooTokenUsage(usage) {
+	const tokenUsage = {
+		...usage.totalTokens !== void 0 ? { total: usage.totalTokens } : {},
+		...usage.inputTokens !== void 0 ? { prompt: usage.inputTokens } : {},
+		...usage.outputTokens !== void 0 ? { completion: usage.outputTokens } : {},
+		...usage.cacheReadTokens !== void 0 ? { cached: usage.cacheReadTokens } : {}
+	};
+	return Object.keys(tokenUsage).length > 0 ? tokenUsage : void 0;
+}
+function runConfiguredAdapterEffect(input) {
+	return Effect.gen(function* () {
+		const adapterId = input.config.adapter;
+		if (!adapterId) return {
+			output: "",
+			error: "agent-skill-evals-provider: config.adapter is required. Use codex-json, claude-code-json, or pi-json."
+		};
+		const adapter = yield* getAdapter(adapterId);
+		if (!adapter) return {
+			output: "",
+			error: `agent-skill-evals-provider: unknown adapter "${adapterId}". Supported adapters: ${DOCUMENTED_ADAPTERS.join(", ")}`
+		};
+		if (!input.config.command) return {
+			output: "",
+			error: "agent-skill-evals-provider: config.command is required for dynamic agent runs"
+		};
+		const command = input.config.command;
+		const environment = yield* Environment;
+		const cwd = yield* environment.cwd;
+		const env = yield* environment.env;
+		const baseDir = input.config.baseDir ?? cwd;
+		const result = yield* adapter.run({
+			command: resolveConfiguredPath(baseDir, expandEnvVars(command, env)),
+			args: (input.config.args ?? []).map((arg) => typeof arg === "string" ? resolveConfiguredPath(baseDir, expandEnvVars(arg, env)) : arg),
+			cwd: input.run.worldPath,
+			prompt: input.prompt,
+			evidence: input.run.evidenceCollector,
+			timeoutMs: input.config.timeoutMs ?? 5 * 6e4,
+			env: {
+				...env,
+				...input.config.isolatedHome ? { HOME: join(input.run.runDir, "agent-home") } : {}
+			}
+		});
+		return {
+			output: result.output,
+			...result.error ? { error: result.error } : {}
+		};
+	});
+}
+function collectFileEvidenceEffect(run, preTree) {
+	return Effect.gen(function* () {
+		const postTree = yield* snapshotTreeEffect(run.worldPath);
+		for (const event of diffTrees(preTree, postTree)) run.evidenceCollector.addFileWrite(event);
+	});
+}
+function persistMetadataEffect(input) {
+	return Effect.gen(function* () {
+		const fs = yield* FileSystem;
+		const durationMs = (yield* Clock.currentTimeMillis) - input.startedAt;
+		input.run.evidenceCollector.setOutput(input.output);
+		input.run.evidenceCollector.setRun({
+			runDir: input.run.runDir,
+			worldPath: input.run.worldPath,
+			fixture: input.fixture,
+			durationMs
+		});
+		const evidencePath = yield* writeEvidenceToEffect(input.run.evidenceCollector, input.run.runDir);
+		const metadata = {
+			runDir: input.run.runDir,
+			worldPath: input.run.worldPath,
+			evidencePath,
+			fixture: input.fixture,
+			skill: input.vars.skill,
+			kind: input.vars.kind,
+			preconditionResults: input.preconditionResults,
+			preconditionsPassed: input.preconditionsPassed,
+			durationMs
+		};
+		yield* fs.writeText(join(input.run.runDir, "agent-skill-evals-meta.json"), JSON.stringify(metadata, null, 2));
+		return metadata;
+	});
+}
+var AgentSkillEvalsProvider = class {
+	config;
+	configError;
+	id;
+	constructor(options = {}) {
+		const config = decodeProviderConfig(options.config ?? {});
+		if ("error" in config) {
+			this.config = {};
+			this.configError = config.error;
+		} else this.config = config;
+		const label = options.id ?? "agent-skill-evals";
+		this.id = () => label;
+	}
+	async callApi(prompt, context = {}) {
+		return Effect.runPromise(this.callApiEffect(prompt, context).pipe(Effect.provide(AgentLiveLayer)));
+	}
+	callApiEffect(prompt, context = {}) {
+		const self = this;
+		return Effect.gen(function* () {
+			if (self.configError) return {
+				output: "",
+				error: self.configError
+			};
+			const startedAt = yield* Clock.currentTimeMillis;
+			const vars = varsFromContext(context);
+			const fixture = vars.fixture;
+			if (!fixture) return {
+				output: "",
+				error: "agent-skill-evals-provider: vars.fixture is required. Set vars.fixture to the fixture directory for this test case."
+			};
+			const prepared = yield* prepareRunEffect(fixture, self.config);
+			if ("error" in prepared) return {
+				output: "",
+				error: prepared.error
+			};
+			const preconditions = yield* runPreconditionsEffect(vars, prepared);
+			const preTree = yield* snapshotTreeEffect(prepared.worldPath);
+			let output = "";
+			let error;
+			if (preconditions.passed) {
+				addNativeSkillEvidenceFromConfig(prepared, self.config, startedAt);
+				const result = yield* runConfiguredAdapterEffect({
+					prompt,
+					run: prepared,
+					config: self.config
+				});
+				output = result.output;
+				error = result.error;
+			}
+			yield* collectFileEvidenceEffect(prepared, preTree);
+			const metadata = yield* persistMetadataEffect({
+				run: prepared,
+				fixture,
+				vars,
+				output,
+				preconditionResults: preconditions.results,
+				preconditionsPassed: preconditions.passed,
+				startedAt
+			}).pipe(Effect.orDie);
+			const usage = promptfooTokenUsage(prepared.evidenceCollector.toSnapshot().usage);
+			return {
+				output,
+				...error ? { error } : {},
+				metadata,
+				...usage ? { tokenUsage: usage } : {}
+			};
+		});
+	}
+};
+//#endregion
+export { writeEvidenceToEffect as a, getRuntimeCheck as c, evidenceFromSnapshot as i, makeWorldHandle as n, decodeEvidenceSnapshotEither as o, EvidenceCollector as r, RuntimeCheckCatalogLive as s, AgentSkillEvalsProvider as t };
+
+//# sourceMappingURL=agent-CM7fIL_C.mjs.map