@alis-build/harness-eval 0.1.0

package/dist/loader-BCnFJ8rm.js

@@ -0,0 +1,717 @@
+import { n as parseCardinality } from "./cardinality-DlE44e-4.js";
+import { readFile, readdir, stat } from "node:fs/promises";
+import { isAbsolute, join, relative, resolve } from "node:path";
+import { parse } from "yaml";
+import { z } from "zod";
+//#region src/config/paths.ts
+/**
+* Resolve relative paths in suite config against the suite file directory.
+*/
+function resolvePath(value, suiteDir) {
+	if (isAbsolute(value) || value.startsWith("~/")) return value;
+	return join(suiteDir, value);
+}
+function resolveClaudeCodePaths(block, suiteDir) {
+	const resolved = { ...block };
+	if (typeof resolved.mcpConfig === "string") resolved.mcpConfig = resolvePath(resolved.mcpConfig, suiteDir);
+	if (Array.isArray(resolved.pluginDirs)) resolved.pluginDirs = resolved.pluginDirs.map((p) => typeof p === "string" ? resolvePath(p, suiteDir) : p);
+	if (Array.isArray(resolved.addDirs)) resolved.addDirs = resolved.addDirs.map((p) => typeof p === "string" ? resolvePath(p, suiteDir) : p);
+	for (const field of [
+		"systemPromptFile",
+		"appendSystemPromptFile",
+		"debugFile"
+	]) {
+		const value = resolved[field];
+		if (typeof value === "string" && !value.trim().startsWith("{")) resolved[field] = resolvePath(value, suiteDir);
+	}
+	if (typeof resolved.settings === "string" && !resolved.settings.trim().startsWith("{")) resolved.settings = resolvePath(resolved.settings, suiteDir);
+	return resolved;
+}
+/** Resolve relative paths in a config layer relative to `suiteDir`. */
+function resolveConfigPaths(config, suiteDir) {
+	if (!config) return void 0;
+	const resolved = { ...config };
+	if (typeof resolved.cwd === "string") resolved.cwd = resolvePath(resolved.cwd, suiteDir);
+	if (resolved.claudeCode && typeof resolved.claudeCode === "object" && !Array.isArray(resolved.claudeCode)) resolved.claudeCode = resolveClaudeCodePaths(resolved.claudeCode, suiteDir);
+	return resolved;
+}
+/** Resolve paths on an entire suite after load. */
+function resolveSuitePaths(suite, suiteFilePath) {
+	const suiteDir = configFileDir(suiteFilePath);
+	suite.defaultConfig = resolveConfigPaths(suite.defaultConfig, suiteDir);
+	for (const cell of suite.matrix) cell.config = resolveConfigPaths(cell.config, suiteDir) ?? cell.config;
+	for (const testCase of suite.cases) testCase.config = resolveConfigPaths(testCase.config, suiteDir);
+}
+function configFileDir(filePath) {
+	return filePath.includes("/") || filePath.includes("\\") ? filePath.replace(/[/\\][^/\\]+$/, "") : ".";
+}
+function resolveEnvPaths(env, baseDir) {
+	const resolved = {};
+	for (const [key, value] of Object.entries(env)) if (value.startsWith("./") || value.startsWith("../") || value.includes("/") && !value.startsWith("http")) resolved[key] = resolvePath(value, baseDir);
+	else resolved[key] = value;
+	return resolved;
+}
+/** Resolve relative paths in a standalone grading config file. */
+function resolveGradingConfigPaths(config, configFilePath) {
+	const baseDir = configFileDir(configFilePath);
+	const { adapter, maxConcurrent, ...rest } = config.judge;
+	config.judge = {
+		...resolveConfigPaths(rest, baseDir) ?? rest,
+		adapter,
+		maxConcurrent
+	};
+	if (config.judge.env) config.judge.env = resolveEnvPaths(config.judge.env, baseDir);
+}
+//#endregion
+//#region src/config/schema.ts
+/**
+* zod schemas for the YAML on-disk shape.
+*
+* Config uses a nested layout: generic harness fields at the top level,
+* adapter-specific options under a named key (e.g. `claudeCode`).
+*/
+/** Claude Code adapter-specific options (nested under `claudeCode`). */
+const ClaudeCodeConfigSchema = z.object({
+	binary: z.string(),
+	pluginDirs: z.array(z.string()),
+	mcpConfig: z.string(),
+	permissionMode: z.enum([
+		"default",
+		"acceptEdits",
+		"plan",
+		"auto",
+		"dontAsk",
+		"bypassPermissions"
+	]),
+	effort: z.enum([
+		"low",
+		"medium",
+		"high",
+		"xhigh",
+		"max"
+	]),
+	pluginUrls: z.array(z.string()),
+	addDirs: z.array(z.string()),
+	strictMcpConfig: z.boolean(),
+	agent: z.string(),
+	fallbackModel: z.string(),
+	tools: z.string(),
+	maxBudgetUsd: z.number().positive(),
+	settings: z.string(),
+	settingSources: z.string(),
+	systemPrompt: z.string(),
+	systemPromptFile: z.string(),
+	appendSystemPrompt: z.string(),
+	appendSystemPromptFile: z.string(),
+	debug: z.union([z.string(), z.boolean()]),
+	debugFile: z.string(),
+	includeHookEvents: z.boolean(),
+	noSessionPersistence: z.boolean(),
+	disableSlashCommands: z.boolean(),
+	bare: z.boolean(),
+	safeMode: z.boolean(),
+	allowDangerouslySkipPermissions: z.boolean(),
+	dangerouslySkipPermissions: z.boolean(),
+	allowedTools: z.array(z.string()),
+	disallowedTools: z.array(z.string()),
+	maxTurns: z.number().int().positive(),
+	isolateConfig: z.boolean()
+}).partial();
+/** Generic + nested adapter config for one layer (defaultConfig, case, cell). */
+const ConfigPartialSchema = z.object({
+	model: z.string(),
+	cwd: z.string(),
+	timeoutMs: z.number().int().positive(),
+	env: z.record(z.string(), z.string()),
+	claudeCode: ClaudeCodeConfigSchema
+}).partial();
+/** A matrix cell — one point in the configuration matrix. */
+const MatrixCellSchema = z.object({
+	label: z.string().min(1),
+	config: ConfigPartialSchema,
+	axes: z.record(z.string(), z.string()).optional()
+});
+/** Reference tool call in suite YAML. */
+const ReferenceToolCallSchema = z.object({
+	tool_name: z.string().min(1),
+	tool_input: z.unknown()
+});
+/** A test case. */
+const TestCaseSchema = z.object({
+	id: z.string().min(1),
+	prompt: z.string().min(1),
+	category: z.string().optional(),
+	notes: z.string().optional(),
+	expectations: z.array(z.string().min(1)).optional(),
+	reference_trajectory: z.array(ReferenceToolCallSchema).optional(),
+	human_ratings: z.record(z.string(), z.number()).optional(),
+	assertions: z.array(z.unknown()).min(1),
+	repetitions: z.number().int().positive().optional(),
+	config: ConfigPartialSchema.optional()
+});
+/** Top-level suite shape. */
+const TestSuiteSchema = z.object({
+	adapter: z.string().optional(),
+	defaultConfig: ConfigPartialSchema.optional(),
+	matrix: z.array(MatrixCellSchema).min(1),
+	cases: z.array(TestCaseSchema).min(1)
+});
+/** Directory suite root (suite.yaml) — cases may live under cases/ as separate YAML files. */
+const SuiteDirectorySchema = z.object({
+	adapter: z.string().optional(),
+	defaultConfig: ConfigPartialSchema.optional(),
+	matrix: z.array(MatrixCellSchema).min(1),
+	cases: z.array(TestCaseSchema).optional()
+});
+//#endregion
+//#region src/config/transform.ts
+/**
+* Thrown when a YAML suite fails to validate or transform. Carries a JSON-path-
+* like trail so users can find the offending node in their config quickly.
+*/
+var ConfigError = class extends Error {
+	path;
+	constructor(message, path) {
+		super(path ? `[${path}] ${message}` : message);
+		this.path = path;
+		this.name = "ConfigError";
+	}
+};
+/** Transform a zod-validated raw suite into the runtime `TestSuite` shape. */
+function transformSuite(raw) {
+	return transformSuiteParts(raw);
+}
+/** Transform a directory `suite.yaml` (cases optional) into runtime shape. */
+function transformSuiteDirectory(raw) {
+	return transformSuiteParts({
+		...raw,
+		cases: raw.cases ?? []
+	});
+}
+/** Transform parsed case files into runtime test cases. */
+function transformTestCases(raw, pathPrefix) {
+	return raw.map((c, i) => transformTestCase(c, `${pathPrefix}[${i}]`));
+}
+function transformSuiteParts(raw) {
+	return {
+		adapter: raw.adapter,
+		defaultConfig: raw.defaultConfig,
+		matrix: raw.matrix.map(transformMatrixCell),
+		cases: raw.cases.map((c, i) => transformTestCase(c, `cases[${i}]`))
+	};
+}
+function transformMatrixCell(raw) {
+	return {
+		label: raw.label,
+		config: raw.config,
+		axes: raw.axes
+	};
+}
+function transformTestCase(raw, path) {
+	return {
+		id: raw.id,
+		prompt: raw.prompt,
+		category: raw.category,
+		notes: raw.notes,
+		expectations: raw.expectations,
+		reference_trajectory: raw.reference_trajectory,
+		human_ratings: raw.human_ratings,
+		repetitions: raw.repetitions,
+		config: raw.config,
+		assertions: raw.assertions.map((a, i) => transformThresholdedAssertion(a, `${path}.assertions[${i}]`))
+	};
+}
+/** Keys that may appear alongside an assertion-type key. Not assertion types themselves. */
+const SIBLING_KEYS = /* @__PURE__ */ new Set(["threshold"]);
+function transformThresholdedAssertion(raw, path) {
+	if (!isPlainObject(raw)) throw new ConfigError(`expected object, got ${typeOf(raw)}`, path);
+	const threshold = raw.threshold;
+	if (threshold !== void 0) {
+		if (typeof threshold !== "number" || threshold < 0 || threshold > 1) throw new ConfigError(`threshold must be a number in [0, 1], got ${JSON.stringify(threshold)}`, `${path}.threshold`);
+	}
+	return {
+		assertion: transformAssertion(raw, path),
+		threshold: typeof threshold === "number" ? threshold : void 0
+	};
+}
+/**
+* Transform one assertion from YAML shape to runtime shape.
+*
+* Finds the single non-sibling key, dispatches to the per-type transformer.
+* Per-type transformers handle both verbose-object and shortcut-scalar input
+* shapes where applicable.
+*/
+function transformAssertion(raw, path) {
+	if (!isPlainObject(raw)) throw new ConfigError(`expected object, got ${typeOf(raw)}`, path);
+	const typeKeys = Object.keys(raw).filter((k) => !SIBLING_KEYS.has(k));
+	if (typeKeys.length === 0) throw new ConfigError(`no assertion type key found (got only sibling keys: ${Object.keys(raw).join(", ")})`, path);
+	if (typeKeys.length > 1) throw new ConfigError(`multiple assertion type keys; pick one: ${typeKeys.join(", ")}`, path);
+	const typeKey = typeKeys[0];
+	const value = raw[typeKey];
+	const valuePath = `${path}.${typeKey}`;
+	switch (typeKey) {
+		case "called": return transformCalled(value, valuePath);
+		case "not_called": return transformNotCalled(value, valuePath);
+		case "called_any_of": return transformCalledAnyOf(value, valuePath);
+		case "called_all_of": return transformCalledAllOf(value, valuePath);
+		case "called_before": return transformCalledBefore(value, valuePath);
+		case "sequence": return transformSequence(value, valuePath);
+		case "called_with": return transformCalledWith(value, valuePath);
+		case "responded_without_tool_calls": return transformRespondedWithoutToolCalls(value, valuePath);
+		case "iterations_within": return transformScalarMax(value, valuePath, "iterations_within");
+		case "cost_within_usd": return transformScalarMax(value, valuePath, "cost_within_usd");
+		case "duration_within_ms": return transformScalarMax(value, valuePath, "duration_within_ms");
+		case "finished_with": return transformFinishedWith(value, valuePath);
+		case "response_contains": return transformResponseText(value, valuePath, "response_contains");
+		case "response_not_contains": return transformResponseText(value, valuePath, "response_not_contains");
+		case "response_matches": return transformResponseMatches(value, valuePath);
+		case "all_of": return transformAllOf(value, valuePath);
+		case "any_of": return transformAnyOf(value, valuePath);
+		case "not": return transformNot(value, valuePath);
+		default: throw new ConfigError(`unknown assertion type: ${typeKey}`, path);
+	}
+}
+function transformCalled(value, path) {
+	if (typeof value === "string") return {
+		type: "called",
+		tool: value
+	};
+	if (!isPlainObject(value)) throw new ConfigError(`expected string or object, got ${typeOf(value)}`, path);
+	const tool = requireToolPattern(value.tool, `${path}.tool`);
+	let times;
+	if (value.times !== void 0) {
+		times = requireString(value.times, `${path}.times`);
+		try {
+			parseCardinality(times);
+		} catch (err) {
+			throw new ConfigError(err instanceof Error ? err.message : `invalid cardinality: ${times}`, `${path}.times`);
+		}
+	}
+	return {
+		type: "called",
+		tool,
+		times
+	};
+}
+function transformNotCalled(value, path) {
+	if (typeof value === "string") return {
+		type: "not_called",
+		tool: value
+	};
+	if (!isPlainObject(value)) throw new ConfigError(`expected string or object, got ${typeOf(value)}`, path);
+	return {
+		type: "not_called",
+		tool: requireToolPattern(value.tool, `${path}.tool`)
+	};
+}
+function transformCalledAnyOf(value, path) {
+	return {
+		type: "called_any_of",
+		tools: requireToolPatternList(value, path)
+	};
+}
+function transformCalledAllOf(value, path) {
+	return {
+		type: "called_all_of",
+		tools: requireToolPatternList(value, path)
+	};
+}
+function transformCalledBefore(value, path) {
+	if (!isPlainObject(value)) throw new ConfigError(`expected object with {first, then}, got ${typeOf(value)}`, path);
+	return {
+		type: "called_before",
+		first: requireToolPattern(value.first, `${path}.first`),
+		then: requireToolPattern(value.then, `${path}.then`)
+	};
+}
+function transformSequence(value, path) {
+	if (Array.isArray(value)) return {
+		type: "sequence",
+		tools: value.map((v, i) => requireToolPattern(v, `${path}[${i}]`))
+	};
+	if (!isPlainObject(value)) throw new ConfigError(`expected array or object, got ${typeOf(value)}`, path);
+	return {
+		type: "sequence",
+		tools: requireToolPatternList(value.tools, `${path}.tools`),
+		strict: value.strict === void 0 ? void 0 : requireBool(value.strict, `${path}.strict`)
+	};
+}
+function transformCalledWith(value, path) {
+	if (!isPlainObject(value)) throw new ConfigError(`expected object with {tool, args}, got ${typeOf(value)}`, path);
+	const tool = requireToolPattern(value.tool, `${path}.tool`);
+	if (value.args === void 0) throw new ConfigError(`missing required field 'args'`, `${path}.args`);
+	validatePredicate(value.args, `${path}.args`);
+	return {
+		type: "called_with",
+		tool,
+		args: value.args
+	};
+}
+function transformRespondedWithoutToolCalls(value, path) {
+	if (value === true || value === null || isPlainObject(value) && Object.keys(value).length === 0) return { type: "responded_without_tool_calls" };
+	throw new ConfigError(`expected true or empty object, got ${JSON.stringify(value)}`, path);
+}
+function transformScalarMax(value, path, type) {
+	let max;
+	if (typeof value === "number") max = value;
+	else if (isPlainObject(value) && typeof value.max === "number") max = value.max;
+	else throw new ConfigError(`expected number or {max: number}, got ${JSON.stringify(value)}`, path);
+	if (max <= 0) throw new ConfigError(`max must be positive, got ${max}`, path);
+	return {
+		type,
+		max
+	};
+}
+function transformFinishedWith(value, path) {
+	if (typeof value === "string") return {
+		type: "finished_with",
+		reasons: value
+	};
+	if (Array.isArray(value)) return {
+		type: "finished_with",
+		reasons: value.map((v, i) => requireString(v, `${path}[${i}]`))
+	};
+	if (isPlainObject(value)) {
+		const reasons = value.reasons;
+		if (typeof reasons === "string") return {
+			type: "finished_with",
+			reasons
+		};
+		if (Array.isArray(reasons)) return {
+			type: "finished_with",
+			reasons: reasons.map((v, i) => requireString(v, `${path}.reasons[${i}]`))
+		};
+	}
+	throw new ConfigError(`expected string, string[], or {reasons: ...}, got ${JSON.stringify(value)}`, path);
+}
+function transformResponseText(value, path, type) {
+	if (typeof value === "string") return {
+		type,
+		text: value
+	};
+	if (isPlainObject(value) && typeof value.text === "string") return {
+		type,
+		text: value.text
+	};
+	throw new ConfigError(`expected string or {text: string}, got ${JSON.stringify(value)}`, path);
+}
+function transformResponseMatches(value, path) {
+	if (!isPlainObject(value)) throw new ConfigError(`expected object with {pattern, flags?}, got ${typeOf(value)}`, path);
+	return {
+		type: "response_matches",
+		pattern: requireString(value.pattern, `${path}.pattern`),
+		flags: value.flags === void 0 ? void 0 : requireString(value.flags, `${path}.flags`)
+	};
+}
+function transformAllOf(value, path) {
+	return {
+		type: "all_of",
+		assertions: transformCompoundList(value, path)
+	};
+}
+function transformAnyOf(value, path) {
+	return {
+		type: "any_of",
+		assertions: transformCompoundList(value, path)
+	};
+}
+function transformNot(value, path) {
+	return {
+		type: "not",
+		assertion: transformAssertion(value, path)
+	};
+}
+function transformCompoundList(value, path) {
+	const list = Array.isArray(value) ? value : isPlainObject(value) && Array.isArray(value.assertions) ? value.assertions : null;
+	if (list === null) throw new ConfigError(`expected array or {assertions: [...]}, got ${JSON.stringify(value)}`, path);
+	return list.map((a, i) => transformAssertion(a, `${path}[${i}]`));
+}
+const LEAF_OPS = /* @__PURE__ */ new Set([
+	"equals",
+	"contains",
+	"not_contains",
+	"regex",
+	"gte",
+	"lte",
+	"gt",
+	"lt",
+	"one_of"
+]);
+const COMPOUND_OPS = /* @__PURE__ */ new Set([
+	"any_of",
+	"all_of",
+	"not"
+]);
+/**
+* Validate that a predicate is well-formed. The runtime engine is tolerant
+* (returns false on bad shapes), but the loader is strict — invalid
+* predicates are far more often user typos than intentional patterns.
+*
+* Permitted shapes:
+*   - scalar (treated as `{equals: scalar}` at runtime)
+*   - single-key object whose key is a leaf op (e.g. `{contains: "x"}`)
+*   - single-key compound (`{any_of: [...]}`, `{all_of: [...]}`, `{not: ...}`)
+*   - multi-key object (descend into fields; each value is a sub-predicate)
+*/
+function validatePredicate(raw, path) {
+	if (!isPlainObject(raw)) return;
+	const keys = Object.keys(raw);
+	if (keys.length === 1) {
+		const key = keys[0];
+		if (LEAF_OPS.has(key)) {
+			validateLeafOperator(key, raw[key], `${path}.${key}`);
+			return;
+		}
+		if (COMPOUND_OPS.has(key)) {
+			if (key === "not") validatePredicate(raw[key], `${path}.not`);
+			else {
+				const arr = raw[key];
+				if (!Array.isArray(arr)) throw new ConfigError(`${key} must be an array, got ${typeOf(arr)}`, `${path}.${key}`);
+				arr.forEach((sub, i) => validatePredicate(sub, `${path}.${key}[${i}]`));
+			}
+			return;
+		}
+	}
+	for (const [field, sub] of Object.entries(raw)) validatePredicate(sub, `${path}.${field}`);
+}
+function validateLeafOperator(op, value, path) {
+	switch (op) {
+		case "equals": return;
+		case "contains":
+		case "not_contains":
+			if (typeof value !== "string") throw new ConfigError(`${op} requires a string`, path);
+			return;
+		case "regex":
+			if (typeof value !== "string") throw new ConfigError("regex requires a string", path);
+			try {
+				new RegExp(value);
+			} catch {
+				throw new ConfigError(`invalid regex: ${value}`, path);
+			}
+			return;
+		case "gte":
+		case "lte":
+		case "gt":
+		case "lt":
+			if (typeof value !== "number") throw new ConfigError(`${op} requires a number`, path);
+			return;
+		case "one_of":
+			if (!Array.isArray(value)) throw new ConfigError("one_of requires an array", path);
+			return;
+		default: return;
+	}
+}
+function requireToolPattern(value, path) {
+	if (typeof value === "string") return value;
+	if (isPlainObject(value) && typeof value.pattern === "string") return { pattern: value.pattern };
+	throw new ConfigError(`expected string or {pattern: string}, got ${JSON.stringify(value)}`, path);
+}
+function requireToolPatternList(value, path) {
+	const list = Array.isArray(value) ? value : isPlainObject(value) && Array.isArray(value.tools) ? value.tools : null;
+	if (list === null) throw new ConfigError(`expected array of tool patterns or {tools: [...]}, got ${JSON.stringify(value)}`, path);
+	return list.map((v, i) => requireToolPattern(v, `${path}[${i}]`));
+}
+function requireString(value, path) {
+	if (typeof value === "string") return value;
+	throw new ConfigError(`expected string, got ${typeOf(value)}`, path);
+}
+function requireBool(value, path) {
+	if (typeof value === "boolean") return value;
+	throw new ConfigError(`expected boolean, got ${typeOf(value)}`, path);
+}
+function isPlainObject(x) {
+	return typeof x === "object" && x !== null && !Array.isArray(x);
+}
+function typeOf(x) {
+	if (x === null) return "null";
+	if (Array.isArray(x)) return "array";
+	return typeof x;
+}
+//#endregion
+//#region src/config/grading-schema.ts
+/**
+* Zod schema for standalone grading YAML (`grading.yaml`).
+*/
+/** Top-level `judge` block — mirrors harness config fields plus grader concurrency. */
+const JudgeConfigSchema = ConfigPartialSchema.extend({
+	adapter: z.string().optional(),
+	maxConcurrent: z.number().int().positive().optional(),
+	/** Optional judge prompt prefix (maps to upstream system_instruction). */
+	system_instruction: z.string().optional()
+});
+const GradingConfigSchema = z.object({ judge: JudgeConfigSchema });
+//#endregion
+//#region src/config/grading-loader.ts
+/**
+* Load standalone grading YAML for `harness-eval grade`.
+*/
+async function loadGradingConfig(filePath) {
+	const absolutePath = resolve(filePath);
+	let content;
+	try {
+		content = await readFile(absolutePath, "utf8");
+	} catch (err) {
+		throw new ConfigError(`failed to read grading config: ${err instanceof Error ? err.message : String(err)}`, filePath);
+	}
+	return parseGradingConfig(content, absolutePath);
+}
+function parseGradingConfig(yamlContent, sourcePath) {
+	let raw;
+	try {
+		raw = parse(yamlContent);
+	} catch (err) {
+		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
+	}
+	const validated = GradingConfigSchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError$1(validated.error, sourcePath)}`, sourcePath);
+	const config = { judge: { ...validated.data.judge } };
+	if (sourcePath) resolveGradingConfigPaths(config, sourcePath);
+	return config;
+}
+function formatZodError$1(err, sourcePath) {
+	return err.issues.map((issue) => {
+		const path = issue.path.length > 0 ? issue.path.join(".") : "(root)";
+		return `  ${sourcePath ? `${sourcePath} → ${path}` : path}: ${issue.message}`;
+	}).join("\n");
+}
+//#endregion
+//#region src/config/loader.ts
+/**
+* Load a `TestSuite` from a YAML file, directory, or string.
+*/
+async function loadSuite(filePath) {
+	const absolutePath = resolve(filePath);
+	let info;
+	try {
+		info = await stat(absolutePath);
+	} catch (err) {
+		throw new ConfigError(`failed to read suite path: ${err instanceof Error ? err.message : String(err)}`, filePath);
+	}
+	if (info.isDirectory()) return loadSuiteDirectory(absolutePath);
+	return loadSuiteFile(absolutePath);
+}
+async function loadSuiteFile(absolutePath) {
+	let content;
+	try {
+		content = await readFile(absolutePath, "utf8");
+	} catch (err) {
+		throw new ConfigError(`failed to read suite file: ${err instanceof Error ? err.message : String(err)}`, absolutePath);
+	}
+	return parseSuite(content, absolutePath);
+}
+async function loadSuiteDirectory(dir) {
+	const suiteYamlPath = join(dir, "suite.yaml");
+	let content;
+	try {
+		content = await readFile(suiteYamlPath, "utf8");
+	} catch (err) {
+		throw new ConfigError(`missing suite.yaml in suite directory: ${err instanceof Error ? err.message : String(err)}`, dir);
+	}
+	const base = parseSuiteDirectory(content, suiteYamlPath);
+	const casesDir = join(dir, "cases");
+	const caseFiles = await collectCaseYamlFiles(casesDir);
+	const tagged = base.cases.map((testCase, index) => ({
+		relPath: "suite.yaml",
+		index,
+		testCase
+	}));
+	for (const filePath of caseFiles) {
+		const cases = parseCasesFile(await readFile(filePath, "utf8"), filePath);
+		const relPath = relative(casesDir, filePath);
+		for (const [index, testCase] of cases.entries()) tagged.push({
+			relPath,
+			index,
+			testCase
+		});
+	}
+	tagged.sort((a, b) => {
+		const pathCmp = a.relPath.localeCompare(b.relPath);
+		if (pathCmp !== 0) return pathCmp;
+		return a.index - b.index;
+	});
+	const cases = tagged.map((entry) => entry.testCase);
+	if (cases.length === 0) throw new ConfigError("suite directory has no test cases", dir);
+	const suite = {
+		...base,
+		cases
+	};
+	resolveSuitePaths(suite, suiteYamlPath);
+	return suite;
+}
+function parseSuite(yamlContent, sourcePath) {
+	let raw;
+	try {
+		raw = parse(yamlContent);
+	} catch (err) {
+		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
+	}
+	const validated = TestSuiteSchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError(validated.error, sourcePath)}`, sourcePath);
+	const suite = transformSuite(validated.data);
+	if (sourcePath) resolveSuitePaths(suite, resolve(sourcePath));
+	return suite;
+}
+function parseSuiteDirectory(yamlContent, sourcePath) {
+	let raw;
+	try {
+		raw = parse(yamlContent);
+	} catch (err) {
+		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
+	}
+	const validated = SuiteDirectorySchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError(validated.error, sourcePath)}`, sourcePath);
+	return transformSuiteDirectory(validated.data);
+}
+/** Parse one case file: single case, array, or `{ cases: [...] }`. */
+function parseCasesFile(yamlContent, sourcePath) {
+	let raw;
+	try {
+		raw = parse(yamlContent);
+	} catch (err) {
+		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
+	}
+	return transformTestCases(extractRawCases(raw, sourcePath), sourcePath ?? "cases");
+}
+function extractRawCases(raw, sourcePath) {
+	if (Array.isArray(raw)) return raw.map((item, index) => validateRawCase(item, sourcePath, index));
+	if (raw && typeof raw === "object") {
+		const obj = raw;
+		if (Array.isArray(obj.cases)) return obj.cases.map((item, index) => validateRawCase(item, sourcePath, index));
+		if ("id" in obj && "prompt" in obj && "assertions" in obj) return [validateRawCase(raw, sourcePath, 0)];
+	}
+	throw new ConfigError("expected a case object, array of cases, or { cases: [...] }", sourcePath);
+}
+function validateRawCase(raw, sourcePath, index) {
+	const validated = TestCaseSchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError(validated.error, sourcePath)}`, sourcePath);
+	return validated.data;
+}
+async function collectCaseYamlFiles(casesDir) {
+	const files = [];
+	async function walk(dir) {
+		let entries;
+		try {
+			entries = await readdir(dir, { withFileTypes: true });
+		} catch (err) {
+			if (err instanceof Error && "code" in err && err.code === "ENOENT") return;
+			throw err;
+		}
+		for (const entry of entries) {
+			const fullPath = join(dir, entry.name);
+			if (entry.isDirectory()) await walk(fullPath);
+			else if (entry.isFile() && (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml"))) files.push(fullPath);
+		}
+	}
+	await walk(casesDir);
+	return files.sort();
+}
+function formatZodError(err, sourcePath) {
+	return err.issues.map((issue) => {
+		const path = issue.path.length > 0 ? issue.path.join(".") : "(root)";
+		return `  ${sourcePath ? `${sourcePath} → ${path}` : path}: ${issue.message}`;
+	}).join("\n");
+}
+//#endregion
+export { parseGradingConfig as a, loadGradingConfig as i, parseCasesFile as n, ConfigError as o, parseSuite as r, loadSuite as t };
+
+//# sourceMappingURL=loader-BCnFJ8rm.js.map