agent-skill-evals 0.1.0 → 0.1.1

package/dist/assertions/index.mjs

@@ -1,66 +1,74 @@
-import { c as parseRuntimeTestFields, n as FileSystem, r as NodeServicesLive, t as Environment } from "../internal-services-5-mRgNls.mjs";
-import { a as writeEvidenceToEffect, c as getRuntimeCheck, i as evidenceFromSnapshot, n as makeWorldHandle, o as decodeEvidenceSnapshotEither, r as EvidenceCollector, s as RuntimeCheckCatalogLive } from "../agent-CM7fIL_C.mjs";
+import { a as EvidenceCollector, c as decodeEvidenceSnapshot, i as makeWorldHandle, l as parseRuntimeTestFields, n as assertMockServicesHealthy, o as evidenceFromSnapshot, r as stopMockServices, s as writeEvidenceTo } from "../agent-B15R7Xlr.mjs";
+import { n as RUNTIME_CHECKS_BY_TYPE } from "../catalog-9Wh-AfRv.mjs";
 import { join } from "node:path";
-import * as Either from "effect/Either";
-import * as Effect from "effect/Effect";
-import * as Schema from "effect/Schema";
+import { readFile } from "node:fs/promises";
 //#region src/assertions/_shared.ts
-function loadMetadataEffect(context) {
-	return Effect.gen(function* () {
-		const direct = context.providerResponse?.metadata;
-		if (direct && typeof direct === "object" && "worldPath" in direct) return direct;
-		const runDir = (yield* (yield* Environment).env).AGENT_SKILL_EVALS_RUN_DIR;
-		if (runDir) {
-			const parsed = yield* (yield* FileSystem).readText(join(runDir, "agent-skill-evals-meta.json")).pipe(Effect.map((buf) => {
-				try {
-					return JSON.parse(buf);
-				} catch {
-					return null;
-				}
-			}), Effect.catchAll(() => Effect.succeed(null)));
-			if (parsed) return parsed;
-		}
+async function loadMetadata(context) {
+	const direct = context.providerResponse?.metadata;
+	if (direct && typeof direct === "object" && "worldPath" in direct) return direct;
+	const runDir = process.env.AGENT_SKILL_EVALS_RUN_DIR;
+	if (runDir) try {
+		const buf = await readFile(join(runDir, "agent-skill-evals-meta.json"), "utf8");
+		return JSON.parse(buf);
+	} catch {
 		return null;
-	});
+	}
+	return null;
 }
-function loadEvidenceEffect(meta) {
-	return Effect.gen(function* () {
-		const buf = yield* (yield* FileSystem).readText(meta.evidencePath).pipe(Effect.mapError((err) => /* @__PURE__ */ new Error(`evidence: failed to read ${meta.evidencePath}: ${err instanceof Error ? err.message : String(err)}`)));
-		const decoded = decodeEvidenceSnapshotEither(yield* Effect.try({
-			try: () => JSON.parse(buf),
-			catch: (err) => /* @__PURE__ */ new Error(`evidence: invalid JSON in ${meta.evidencePath}: ${err instanceof Error ? err.message : String(err)}`)
-		}));
-		if (Either.isLeft(decoded)) return yield* Effect.fail(/* @__PURE__ */ new Error(`evidence: invalid agent-skill-evals.evidence.v1 payload: ${decoded.left.message}`));
-		const snapshot = decoded.right;
-		return EvidenceCollector.fromSnapshot(snapshot);
-	});
+async function loadEvidence(meta) {
+	let buf;
+	try {
+		buf = await readFile(meta.evidencePath, "utf8");
+	} catch (err) {
+		throw new Error(`evidence: failed to read ${meta.evidencePath}: ${err instanceof Error ? err.message : String(err)}`);
+	}
+	let parsed;
+	try {
+		parsed = JSON.parse(buf);
+	} catch (err) {
+		throw new Error(`evidence: invalid JSON in ${meta.evidencePath}: ${err instanceof Error ? err.message : String(err)}`);
+	}
+	const decoded = decodeEvidenceSnapshot(parsed);
+	if (!decoded.ok) throw new Error(`evidence: ${decoded.error.message}`);
+	return EvidenceCollector.fromSnapshot(decoded.value);
 }
 function loadWorld(meta, evidenceCollector) {
-	return makeWorldHandle(meta.worldPath, (event) => evidenceCollector.addCommand(event));
+	const world = makeWorldHandle(meta.worldPath, (event) => evidenceCollector.addCommand(event));
+	if (!meta.mockEnv) return world;
+	return {
+		...world,
+		exec(command, args, opts = {}) {
+			return world.exec(command, args, {
+				...opts,
+				env: {
+					...meta.mockEnv,
+					...opts.env ?? {}
+				}
+			});
+		}
+	};
 }
-function runEntriesEffect(entries, world, evidenceCollector, mode) {
-	return Effect.gen(function* () {
-		const results = [];
-		for (const entry of entries) {
-			const plugin = yield* getRuntimeCheck(entry.type);
-			if (!plugin) {
-				results.push({
-					pass: false,
-					score: 0,
-					reason: `unknown effect type: ${entry.type}`
-				});
-				continue;
-			}
-			const r = yield* plugin.verify({
-				assertion: entry.args,
-				world,
-				evidence: evidenceFromSnapshot(evidenceCollector.toSnapshot()),
-				mode
+async function runEntries(entries, world, evidenceCollector, mode) {
+	const results = [];
+	for (const entry of entries) {
+		const plugin = RUNTIME_CHECKS_BY_TYPE.get(entry.type);
+		if (!plugin) {
+			results.push({
+				pass: false,
+				score: 0,
+				reason: `unknown effect type: ${entry.type}`
 			});
-			results.push(r);
+			continue;
 		}
-		return results;
-	});
+		const r = await plugin.verify({
+			assertion: entry.args,
+			world,
+			evidence: evidenceFromSnapshot(evidenceCollector.toSnapshot()),
+			mode
+		});
+		results.push(r);
+	}
+	return results;
 }
 function aggregate(results, emptyReason, options = {}) {
 	if (results.length === 0) {
@@ -86,47 +94,82 @@ function aggregate(results, emptyReason, options = {}) {
 }
 //#endregion
 //#region src/assertions/skill-test.ts
-async function skillTest(_output, context) {
-	return Effect.runPromise(skillTestEffect(_output, context).pipe(Effect.provide(RuntimeCheckCatalogLive), Effect.provide(NodeServicesLive)));
+function withEvidencePointer(result, meta) {
+	if (result.pass) return result;
+	return {
+		...result,
+		reason: `${result.reason}\nEvidence: ${meta.evidencePath}\nWorld: ${meta.worldPath}`
+	};
 }
-function skillTestEffect(_output, context) {
-	return Effect.gen(function* () {
-		const meta = yield* loadMetadataEffect(context);
-		if (!meta) return {
+async function skillTest(_output, context) {
+	const meta = await loadMetadata(context);
+	if (!meta) return {
+		pass: false,
+		score: 0,
+		reason: "skill.test: provider metadata missing"
+	};
+	const preconditionResults = meta.preconditionResults ?? [];
+	if (!meta.preconditionsPassed) {
+		const result = withEvidencePointer(aggregate(preconditionResults, "skill.test: preconditions failed"), meta);
+		await stopMockServices(meta.runDir);
+		return result;
+	}
+	const vars = context.vars ?? context.test?.vars ?? {};
+	let evidenceCollector;
+	try {
+		evidenceCollector = await loadEvidence(meta);
+	} catch (err) {
+		await stopMockServices(meta.runDir);
+		return {
 			pass: false,
 			score: 0,
-			reason: "skill.test: provider metadata missing"
+			reason: err instanceof Error ? err.message : String(err)
 		};
-		const preconditionResults = meta.preconditionResults ?? [];
-		if (!meta.preconditionsPassed) return aggregate(preconditionResults, "skill.test: preconditions failed");
-		const vars = context.vars ?? context.test?.vars ?? {};
-		const loadedEvidence = yield* Effect.either(loadEvidenceEffect(meta));
-		if (Either.isLeft(loadedEvidence)) {
-			const err = loadedEvidence.left;
-			return {
-				pass: false,
-				score: 0,
-				reason: err instanceof Error ? err.message : String(err)
-			};
-		}
-		const evidenceCollector = loadedEvidence.right;
-		const world = loadWorld(meta, evidenceCollector);
-		const parsed = parseRuntimeTestFields(vars);
-		const parseResults = parsed.errors.filter((error) => error.field !== "preconditions").map((error) => ({
+	}
+	try {
+		assertMockServicesHealthy(meta.runDir);
+	} catch (err) {
+		await stopMockServices(meta.runDir);
+		return withEvidencePointer({
 			pass: false,
 			score: 0,
-			reason: `runtime test field ${error.index === void 0 ? error.field : `${error.field}[${error.index}]`}: ${error.reason}`
-		}));
-		const shouldResults = yield* runEntriesEffect(parsed.should, world, evidenceCollector, "should");
-		const shouldNotResults = yield* runEntriesEffect(parsed.should_not, world, evidenceCollector, "should_not");
-		yield* writeEvidenceToEffect(evidenceCollector, meta.runDir).pipe(Effect.orDie);
-		return aggregate([
-			...preconditionResults,
-			...parseResults,
-			...shouldResults,
-			...shouldNotResults
-		], "skill.test: no Runtime Test Fields checks declared", { emptyPass: false });
-	});
+			reason: `Mock Service failure: ${err instanceof Error ? err.message : String(err)}`
+		}, meta);
+	}
+	const world = loadWorld(meta, evidenceCollector);
+	const warningResults = evidenceCollector.toSnapshot().warnings?.map((warning) => ({
+		pass: false,
+		score: 0,
+		reason: `evidence warning: ${warning}`
+	})) ?? [];
+	const parsed = parseRuntimeTestFields(vars);
+	const parseResults = parsed.errors.filter((error) => error.field !== "preconditions").map((error) => ({
+		pass: false,
+		score: 0,
+		reason: `runtime test field ${error.index === void 0 ? error.field : `${error.field}[${error.index}]`}: ${error.reason}`
+	}));
+	const expectationResults = await runEntries(parsed.expect, world, evidenceCollector, "should");
+	let writeError;
+	try {
+		await writeEvidenceTo(evidenceCollector, meta.runDir);
+	} catch (err) {
+		writeError = err instanceof Error ? err.message : String(err);
+	}
+	const graded = withEvidencePointer(aggregate([
+		...preconditionResults,
+		...warningResults,
+		...parseResults,
+		...expectationResults
+	], "skill.test: no Runtime Test Fields checks declared", { emptyPass: false }), meta);
+	if (writeError !== void 0) {
+		await stopMockServices(meta.runDir);
+		return {
+			...graded,
+			reason: `${graded.reason}\n(note: failed to update evidence.json: ${writeError})`
+		};
+	}
+	await stopMockServices(meta.runDir);
+	return graded;
 }
 //#endregion
 //#region src/assertions/skill-budget.ts
@@ -136,20 +179,20 @@ const budgetFields = [
 	["completion", "maxCompletionTokens"],
 	["cached", "maxCachedTokens"]
 ];
-function configObject$1(value) {
+function configObject(value) {
 	return value && typeof value === "object" && !Array.isArray(value) ? value : {};
 }
-function numberSetting$1(config, key) {
+function numberSetting(config, key) {
 	const value = config[key];
 	return typeof value === "number" && Number.isFinite(value) ? value : void 0;
 }
 function budgetSettings(context) {
-	const config = configObject$1(context.config?.agentSkillEvals);
+	const config = configObject(context.config?.agentSkillEvals);
 	return {
-		maxTotalTokens: numberSetting$1(config, "maxTotalTokens"),
-		maxPromptTokens: numberSetting$1(config, "maxPromptTokens"),
-		maxCompletionTokens: numberSetting$1(config, "maxCompletionTokens"),
-		maxCachedTokens: numberSetting$1(config, "maxCachedTokens")
+		maxTotalTokens: numberSetting(config, "maxTotalTokens"),
+		maxPromptTokens: numberSetting(config, "maxPromptTokens"),
+		maxCompletionTokens: numberSetting(config, "maxCompletionTokens"),
+		maxCachedTokens: numberSetting(config, "maxCachedTokens")
 	};
 }
 function tokenUsage(context) {
@@ -193,379 +236,51 @@ async function skillBudget(_output, context) {
 	};
 }
 //#endregion
-//#region src/skill-checks/assertions-static/_shared.ts
-function getStaticMeta(context) {
-	const m = context.providerResponse?.metadata;
-	if (m && typeof m === "object" && ("skill" in m || "tests" in m)) return m;
-	return null;
-}
-function pass(reason, components) {
-	return {
-		pass: true,
-		score: 1,
-		reason,
-		componentResults: components
-	};
-}
-function fail(reason, components) {
-	return {
-		pass: false,
-		score: 0,
-		reason,
-		componentResults: components
-	};
-}
-//#endregion
-//#region src/skill-checks/assertions-static/settings.ts
-const DEFAULT_SETTINGS = {
-	maxSkillLines: 200,
-	destructiveEffects: ["file.changes_outside_scope", "tool.called"],
-	requireTokenBudget: false,
-	riskyEffects: ["file.changes_outside_scope", "tool.called"]
+//#region src/assertions/promptfoo.ts
+const routes = {
+	"skill.budget": skillBudget,
+	"skill.test": skillTest
 };
-const StringArraySchema = Schema.Array(Schema.String);
-function configObject(value) {
-	return value && typeof value === "object" && !Array.isArray(value) ? value : {};
-}
-function numberSetting(config, key, fallback) {
-	return Schema.is(Schema.Number)(config[key]) ? config[key] : fallback;
-}
-function stringArraySetting(config, key, fallback) {
-	return Schema.is(StringArraySchema)(config[key]) ? config[key] : fallback;
-}
-function booleanSetting(config, key, fallback) {
-	return typeof config[key] === "boolean" ? config[key] : fallback;
-}
-function skillCheckSettings(context) {
-	const config = configObject(context.config?.agentSkillEvals);
-	return {
-		maxSkillLines: numberSetting(config, "maxSkillLines", DEFAULT_SETTINGS.maxSkillLines),
-		destructiveEffects: stringArraySetting(config, "destructiveEffects", DEFAULT_SETTINGS.destructiveEffects),
-		requireTokenBudget: booleanSetting(config, "requireTokenBudget", DEFAULT_SETTINGS.requireTokenBudget),
-		riskyEffects: stringArraySetting(config, "riskyEffects", DEFAULT_SETTINGS.riskyEffects)
-	};
-}
-//#endregion
-//#region src/skill-checks/assertions-static/context-economy.ts
-/**
-* SPEC §7.2 — context economy. Hard: referenced files must exist (already
-* surfaced via missingFiles). Warnings: SKILL.md size, reference depth,
-* missing TOC in long reference files.
-*
-* Implemented as warnings-only: this assertion always passes, but emits a
-* `score < 1` reason describing how SKILL.md compares to thresholds.
-*/
-async function contextEconomy(_output, context) {
-	const meta = getStaticMeta(context);
-	if (!meta) return fail("context-economy: provider metadata missing");
-	const skill = meta.skill;
-	if (!skill) return fail("context-economy: skill not parsed");
-	const { maxSkillLines: maxLines } = skillCheckSettings(context);
-	const components = [];
-	components.push({
-		pass: skill.totalLines <= maxLines,
-		score: skill.totalLines <= maxLines ? 1 : .5,
-		reason: `SKILL.md ${skill.totalLines} line(s) (limit ${maxLines})`
-	});
-	components.push({
-		pass: skill.missingReferences.length === 0,
-		score: skill.missingReferences.length === 0 ? 1 : 0,
-		reason: skill.missingReferences.length === 0 ? `${skill.references.length} reference(s) all resolved` : `missing references: ${skill.missingReferences.slice(0, 3).join(", ")}`
-	});
-	const failed = components.filter((c) => !c.pass && c.reason.startsWith("missing references"));
-	if (failed.length === 0) {
-		const oversize = components.find((c) => c.score < 1);
-		return oversize ? {
-			pass: true,
-			score: .5,
-			reason: oversize.reason,
-			componentResults: components
-		} : pass(`context-economy: ok`, components);
+const knownMetrics = Object.keys(routes);
+const availableMetrics = [...knownMetrics].sort().join(", ");
+function sameConfig(a, b) {
+	if (a === void 0 && b === void 0) return true;
+	try {
+		return JSON.stringify(a) === JSON.stringify(b);
+	} catch {
+		return false;
 	}
-	return fail(failed.map((f) => `✗ ${f.reason}`).join("; "), components);
-}
-//#endregion
-//#region src/skill-checks/assertions-static/executable-helper.ts
-/**
-* SPEC §7.4 — verifier scripts referenced by tests must exist and be
-* runnable. Generated tests must not reference missing fixtures.
-*/
-async function executableHelper(_output, context) {
-	const meta = getStaticMeta(context);
-	if (!meta) return fail("executable-helper: provider metadata missing");
-	const tests = meta.tests;
-	if (!tests) return fail("executable-helper: tests not parsed");
-	const components = [];
-	components.push({
-		pass: tests.missingVerifierScripts.length === 0,
-		score: tests.missingVerifierScripts.length === 0 ? 1 : 0,
-		reason: tests.missingVerifierScripts.length === 0 ? `${tests.verifierScripts.length} verifier script(s) all present` : `missing verifier scripts: ${tests.missingVerifierScripts.slice(0, 3).join(", ")}`
-	});
-	components.push({
-		pass: tests.nonExecutableVerifierScripts.length === 0,
-		score: tests.nonExecutableVerifierScripts.length === 0 ? 1 : 0,
-		reason: tests.nonExecutableVerifierScripts.length === 0 ? `${tests.verifierScripts.length} verifier script(s) executable` : `non-executable verifier scripts: ${tests.nonExecutableVerifierScripts.slice(0, 3).join(", ")}`
-	});
-	components.push({
-		pass: tests.missingFixturePaths.length === 0,
-		score: tests.missingFixturePaths.length === 0 ? 1 : 0,
-		reason: tests.missingFixturePaths.length === 0 ? `${tests.fixturePaths.length} fixture(s) all present` : `missing fixtures: ${tests.missingFixturePaths.slice(0, 3).join(", ")}`
-	});
-	const failed = components.filter((c) => !c.pass);
-	if (failed.length === 0) return pass("executable-helper: ok", components);
-	return fail(failed.map((f) => `✗ ${f.reason}`).join("; "), components);
-}
-//#endregion
-//#region src/skill-checks/assertions-static/instruction-calibration.ts
-/**
-* SPEC §7.3 — for skills whose test pack uses destructive effects, the
-* SKILL.md must contain confirmation/clarification language and at least
-* one negative test must declare forbidden effects.
-*
-* `assert.config.agentSkillEvals.destructiveEffects` overrides the default list.
-*/
-const CONFIRMATION_RE = /\b(confirm|ask first|do not.*without|before.*push|require.*approval|do not.*destructive)\b/i;
-const PLAN_BEFORE_ACT_RE = /\b(plan first|plan before|read.*before.*write|validate.*before|dry.run)\b/i;
-async function instructionCalibration(_output, context) {
-	const meta = getStaticMeta(context);
-	if (!meta) return fail("skill.instructions: provider metadata missing");
-	const skill = meta.skill;
-	const tests = meta.tests;
-	if (!skill) return fail("skill.instructions: skill not parsed");
-	const destructive = new Set(skillCheckSettings(context).destructiveEffects);
-	if (!(tests ? tests.tests.some((t) => t.effectTypes.some((e) => destructive.has(e))) : false)) return pass("skill.instructions: no destructive effects in test pack");
-	const components = [];
-	const hasConfirm = CONFIRMATION_RE.test(skill.body) || PLAN_BEFORE_ACT_RE.test(skill.body);
-	components.push({
-		pass: hasConfirm,
-		score: hasConfirm ? 1 : 0,
-		reason: hasConfirm ? "SKILL.md describes confirmation / plan-before-act" : "SKILL.md uses destructive effects but lacks confirmation / plan-before-act language"
-	});
-	const declaresForbidden = tests ? tests.tests.some((t) => Array.isArray(t.vars.should_not) && t.vars.should_not.length > 0) : false;
-	components.push({
-		pass: declaresForbidden,
-		score: declaresForbidden ? 1 : 0,
-		reason: declaresForbidden ? "test pack declares forbidden effects (should_not)" : "no should_not declared in any test, despite destructive effects"
-	});
-	const failed = components.filter((c) => !c.pass);
-	if (failed.length === 0) return pass("skill.instructions: ok", components);
-	return fail(failed.map((f) => `✗ ${f.reason}`).join("; "), components);
-}
-//#endregion
-//#region src/skill-checks/assertions-static/negative-coverage.ts
-/**
-* SPEC §7.6 — risky skills (those whose test pack uses any risky effect)
-* must include at least one negative test.
-*
-* `assert.config.agentSkillEvals.riskyEffects` overrides the default risky-effect list.
-*/
-async function negativeCoverage(_output, context) {
-	const meta = getStaticMeta(context);
-	if (!meta) return fail("skill.tests: provider metadata missing");
-	const tests = meta.tests;
-	if (!tests) return fail("skill.tests: tests not parsed");
-	const risky = new Set(skillCheckSettings(context).riskyEffects);
-	if (!tests.tests.some((t) => t.effectTypes.some((e) => risky.has(e)))) return pass("skill.tests: no risky effects, negative test not required");
-	const negatives = tests.tests.filter((t) => t.isNegative);
-	if (negatives.length === 0) return fail(`skill.tests: skill uses risky effects (${[...risky].join(", ")}) but no negative test (kind: negative) is declared`);
-	return pass(`skill.tests: ${negatives.length} negative test(s)`);
-}
-//#endregion
-//#region src/skill-checks/assertions-static/routing-metadata.ts
-const GENERIC_PHRASES = [
-	/\bhelp(s|ing)?\b/i,
-	/\bgithub workflows?\b/i,
-	/\bvarious\b/i,
-	/\bany kind of\b/i
-];
-const WHEN_TO_USE_RE = /\b(use when|use this|when (?:the )?(?:user|you))\b/i;
-const WHEN_NOT_RE = /\bdo not use|do not invoke|don'?t use|not for|avoid using\b/i;
-/**
-* SPEC §7.1 — routing metadata hard checks.
-*/
-async function routingMetadata(_output, context) {
-	const meta = getStaticMeta(context);
-	if (!meta) return fail("routing-metadata: provider metadata missing");
-	const skill = meta.skill;
-	if (!skill) return fail("routing-metadata: skill not parsed (vars.skillPath missing?)");
-	const fm = skill.frontmatter;
-	const components = [];
-	const name = typeof fm.name === "string" ? fm.name.trim() : "";
-	components.push({
-		pass: name.length > 0,
-		score: name.length > 0 ? 1 : 0,
-		reason: name ? `name: ${name}` : "missing `name` frontmatter"
-	});
-	const desc = typeof fm.description === "string" ? fm.description.trim() : "";
-	components.push({
-		pass: desc.length > 0,
-		score: desc.length > 0 ? 1 : 0,
-		reason: desc ? `description present (${desc.length} chars)` : "missing `description` frontmatter"
-	});
-	const sayWhen = WHEN_TO_USE_RE.test(desc);
-	components.push({
-		pass: sayWhen,
-		score: sayWhen ? 1 : 0,
-		reason: sayWhen ? "description says when to use" : "description does not say when to use (e.g. 'Use when …')"
-	});
-	const sayWhenNot = WHEN_NOT_RE.test(desc);
-	components.push({
-		pass: sayWhenNot,
-		score: sayWhenNot ? 1 : 0,
-		reason: sayWhenNot ? "description says when not to use" : "description does not say when not to use (e.g. 'Do not use for …')"
-	});
-	const generic = GENERIC_PHRASES.some((re) => re.test(desc)) && desc.length < 80;
-	components.push({
-		pass: !generic,
-		score: generic ? 0 : 1,
-		reason: generic ? "description is too generic" : "description is specific enough"
-	});
-	const failed = components.filter((c) => !c.pass);
-	if (failed.length === 0) return pass(`routing-metadata: ${components.length}/${components.length} checks ok`, components);
-	return fail(failed.map((f) => `✗ ${f.reason}`).join("; "), components);
 }
-//#endregion
-//#region src/skill-checks/assertions-static/scenario-validity.ts
 /**
-* SPEC §7.6 — every runtime test must have prompt + fixture (or fixtureless)
-* + at least one of should/should_not. Unsupported effect types fail
-* static validation.
+* Promptfoo hands a javascript assertion its `config` but not its `metric`,
+* so a lone `metric:` declaration used to require a duplicate `config.metric`.
+* Recover the metric from the test's assert list instead: the entry whose
+* config matches the one Promptfoo passed is the assertion being graded.
 */
-async function scenarioValidity(_output, context) {
-	const meta = getStaticMeta(context);
-	if (!meta) return fail("skill.tests: provider metadata missing");
-	const tests = meta.tests;
-	if (!tests) return fail("skill.tests: tests not parsed (vars.testsGlob missing?)");
-	const settings = skillCheckSettings(context);
-	const components = [];
-	if (tests.parseErrors.length > 0) components.push({
-		pass: false,
-		score: 0,
-		reason: `parse errors: ${tests.parseErrors.map((e) => `${e.filePath}: ${e.error}`).join("; ")}`
-	});
-	if (tests.matchedFiles.length === 0) components.push({
-		pass: false,
-		score: 0,
-		reason: "testsGlob matched no test files"
-	});
-	for (const t of tests.tests) {
-		const issues = [];
-		for (const error of t.entryErrors) {
-			const at = error.index === void 0 ? error.field : `${error.field}[${error.index}]`;
-			issues.push(`${at}: ${error.reason}`);
-		}
-		if (typeof t.vars.prompt !== "string" || t.vars.prompt.length === 0) issues.push("missing vars.prompt");
-		if (!t.hasFixture) issues.push("missing vars.fixture (or vars.fixtureless: true)");
-		if (!(Array.isArray(t.vars.should) && t.vars.should.length > 0 || Array.isArray(t.vars.should_not) && t.vars.should_not.length > 0)) issues.push("no should / should_not");
-		if (settings.requireTokenBudget && !t.hasTokenBudget) issues.push("missing skill.budget assertion");
-		components.push({
-			pass: issues.length === 0,
-			score: issues.length === 0 ? 1 : 0,
-			reason: `${t.description ?? "(no description)"}: ${issues.length === 0 ? "ok" : issues.join(", ")}`
-		});
-	}
-	if (meta.unresolvedEffectTypes.length > 0) components.push({
-		pass: false,
-		score: 0,
-		reason: `unsupported effect types: ${meta.unresolvedEffectTypes.join(", ")}`
-	});
-	if (meta.missingFiles.length > 0) components.push({
-		pass: false,
-		score: 0,
-		reason: `missing referenced files: ${meta.missingFiles.slice(0, 5).join(", ")}`
-	});
-	const failed = components.filter((c) => !c.pass);
-	if (failed.length === 0) return pass(`skill.tests: ${tests.tests.length} test(s) ok`, components);
-	return fail(failed.map((f) => `✗ ${f.reason}`).join("; "), components);
-}
-//#endregion
-//#region src/skill-checks/assertions-static/promptfoo.ts
-const metricChecks = {
-	"skill.activation": [routingMetadata],
-	"skill.budgets": [scenarioValidity],
-	"skill.context": [contextEconomy],
-	"skill.instructions": [instructionCalibration],
-	"skill.tests": [scenarioValidity, negativeCoverage],
-	"skill.verifiers": [executableHelper]
-};
-metricChecks["skill.checks"] = [
-	routingMetadata,
-	contextEconomy,
-	instructionCalibration,
-	scenarioValidity,
-	negativeCoverage,
-	executableHelper
-];
-const availableMetrics$1 = Object.keys(metricChecks).sort().join(", ");
-function metricFrom$1(context) {
-	return [
-		context.assertion?.metric,
-		context.assert?.metric,
-		context.config?.metric,
-		context.metric
-	].find((metric) => typeof metric === "string");
-}
-function runChecksEffect(output, context, checks) {
-	return Effect.gen(function* () {
-		const results = yield* Effect.forEach(checks, (check) => Effect.promise(() => check(output, context)), { concurrency: "unbounded" });
-		const failed = results.filter((result) => !result.pass);
-		const soft = results.filter((result) => result.pass && result.score < 1);
-		return {
-			pass: failed.length === 0,
-			score: failed.length === 0 ? Math.min(...results.map((result) => result.score)) : 0,
-			reason: failed.length === 0 ? soft.length > 0 ? soft.map((result) => result.reason).join("; ") : `skill checks: ${results.length} check(s) passed` : failed.map((result) => result.reason).join("; "),
-			componentResults: results
-		};
-	});
-}
-async function agentSkillEvalsStaticAssertions(output, context) {
-	return Effect.runPromise(agentSkillEvalsStaticAssertionsEffect(output, context));
-}
-function agentSkillEvalsStaticAssertionsEffect(output, context) {
-	return Effect.gen(function* () {
-		const metric = metricFrom$1(context);
-		const checks = metric ? metricChecks[metric] : void 0;
-		if (!checks) return {
-			pass: false,
-			score: 0,
-			reason: `agent-skill-evals skill checks: unknown metric "${metric ?? "missing"}". Available metrics: ${availableMetrics$1}`
-		};
-		return yield* runChecksEffect(output, context, checks);
-	});
+function metricFromTestAsserts(context) {
+	const asserts = context.test?.assert;
+	if (!Array.isArray(asserts)) return void 0;
+	const candidates = asserts.filter((entry) => typeof entry === "object" && entry !== null && typeof entry.metric === "string" && knownMetrics.includes(entry.metric));
+	if (candidates.length === 1) return candidates[0].metric;
+	const matched = candidates.filter((entry) => sameConfig(entry.config, context.config));
+	if (matched.length === 1) return matched[0].metric;
 }
-//#endregion
-//#region src/assertions/promptfoo.ts
-const routes = {
-	"skill.budget": skillBudget,
-	"skill.test": skillTest
-};
-const staticMetrics = [
-	"skill.checks",
-	"skill.activation",
-	"skill.budgets",
-	"skill.context",
-	"skill.instructions",
-	"skill.tests",
-	"skill.verifiers"
-];
-const availableMetrics = [...Object.keys(routes), ...staticMetrics].sort().join(", ");
 function metricFrom(context) {
 	return [
 		context.assertion?.metric,
 		context.assert?.metric,
 		context.config?.metric,
 		context.metric
-	].find((metric) => typeof metric === "string");
+	].find((metric) => typeof metric === "string") ?? metricFromTestAsserts(context);
 }
 async function agentSkillEvalsAssertions(output, context) {
 	const metric = metricFrom(context);
 	const assertion = metric ? routes[metric] : void 0;
 	if (assertion) return assertion(output, context);
-	if (metric && staticMetrics.includes(metric)) return agentSkillEvalsStaticAssertions(output, context);
 	return {
 		pass: false,
 		score: 0,
-		reason: `agent-skill-evals assertions: unknown metric "${metric ?? "missing"}". Available metrics: ${availableMetrics}`
+		reason: `agent-skill-evals assertions: unknown metric "${metric ?? "missing"}". Available metrics: ${availableMetrics}. Declare the metric on the assertion (metric: skill.test). If one test case uses several agent-skill-evals assertions with identical config blocks, disambiguate with config.metric.`
 	};
 }
 //#endregion