@alis-build/harness-eval 0.1.2 → 0.1.3

package/dist/{loader-DcI0KfRX.js → loader-DnQ6Jt0i.js}

@@ -3,80 +3,6 @@ import { readFile, readdir, stat } from "node:fs/promises";
 import { isAbsolute, join, relative, resolve } from "node:path";
 import { parse } from "yaml";
 import { z } from "zod";
-//#region src/config/paths.ts
-/**
-* Resolve relative paths in suite config against the suite file directory.
-*
-* YAML authors write paths relative to the suite file; this module absolutizes
-* them at load time so the runner and adapters receive filesystem-ready values.
-* Tilde-prefixed paths and inline JSON blobs (settings starting with `{`) are
-* left unchanged.
-*/
-/** Resolve a single path relative to `suiteDir` unless already absolute or `~/`. */
-function resolvePath(value, suiteDir) {
-	if (isAbsolute(value) || value.startsWith("~/")) return value;
-	return join(suiteDir, value);
-}
-/** Resolve Claude Code-specific path fields within a config block. */
-function resolveClaudeCodePaths(block, suiteDir) {
-	const resolved = { ...block };
-	if (typeof resolved.mcpConfig === "string") resolved.mcpConfig = resolvePath(resolved.mcpConfig, suiteDir);
-	if (Array.isArray(resolved.pluginDirs)) resolved.pluginDirs = resolved.pluginDirs.map((p) => typeof p === "string" ? resolvePath(p, suiteDir) : p);
-	if (Array.isArray(resolved.addDirs)) resolved.addDirs = resolved.addDirs.map((p) => typeof p === "string" ? resolvePath(p, suiteDir) : p);
-	for (const field of [
-		"systemPromptFile",
-		"appendSystemPromptFile",
-		"debugFile"
-	]) {
-		const value = resolved[field];
-		if (typeof value === "string" && !value.trim().startsWith("{")) resolved[field] = resolvePath(value, suiteDir);
-	}
-	if (typeof resolved.settings === "string" && !resolved.settings.trim().startsWith("{")) resolved.settings = resolvePath(resolved.settings, suiteDir);
-	return resolved;
-}
-/** Resolve relative paths in a config layer relative to `suiteDir`. */
-function resolveConfigPaths(config, suiteDir) {
-	if (!config) return void 0;
-	const resolved = { ...config };
-	if (typeof resolved.cwd === "string") resolved.cwd = resolvePath(resolved.cwd, suiteDir);
-	if (resolved.claudeCode && typeof resolved.claudeCode === "object" && !Array.isArray(resolved.claudeCode)) resolved.claudeCode = resolveClaudeCodePaths(resolved.claudeCode, suiteDir);
-	return resolved;
-}
-/** Resolve paths on an entire suite after load. */
-function resolveSuitePaths(suite, suiteFilePath) {
-	const suiteDir = configFileDir(suiteFilePath);
-	suite.defaultConfig = resolveConfigPaths(suite.defaultConfig, suiteDir);
-	for (const cell of suite.matrix) cell.config = resolveConfigPaths(cell.config, suiteDir) ?? cell.config;
-	for (const testCase of suite.cases) testCase.config = resolveConfigPaths(testCase.config, suiteDir);
-}
-/** Parent directory of a suite or grading config file path. */
-function configFileDir(filePath) {
-	return filePath.includes("/") || filePath.includes("\\") ? filePath.replace(/[/\\][^/\\]+$/, "") : ".";
-}
-/**
-* Heuristically resolve env var values that look like relative file paths.
-*
-* Used for grading config where credential or config paths may be expressed
-* relative to the grading YAML location.
-*/
-function resolveEnvPaths(env, baseDir) {
-	const resolved = {};
-	for (const [key, value] of Object.entries(env)) if (value.startsWith("./") || value.startsWith("../") || value.includes("/") && !value.startsWith("http")) resolved[key] = resolvePath(value, baseDir);
-	else resolved[key] = value;
-	return resolved;
-}
-/** Resolve relative paths in a standalone grading config file. */
-function resolveGradingConfigPaths(config, configFilePath) {
-	const baseDir = configFileDir(configFilePath);
-	const { adapter, maxConcurrent, ...rest } = config.judge;
-	config.judge = {
-		...resolveConfigPaths(rest, baseDir) ?? rest,
-		adapter,
-		maxConcurrent
-	};
-	if (config.judge.env) config.judge.env = resolveEnvPaths(config.judge.env, baseDir);
-}
-//#endregion
 //#region src/config/schema.ts
 /**
 * zod schemas for the YAML on-disk shape.
@@ -132,13 +58,40 @@ const ClaudeCodeConfigSchema = z.object({
 	maxTurns: z.number().int().positive(),
 	isolateConfig: z.boolean()
 }).partial();
+/** Codex CLI adapter-specific options (nested under `codex`). */
+const CodexConfigSchema = z.object({
+	binary: z.string(),
+	profile: z.string(),
+	sandbox: z.enum([
+		"read-only",
+		"workspace-write",
+		"danger-full-access"
+	]),
+	addDirs: z.array(z.string()),
+	configOverrides: z.array(z.string()),
+	askForApproval: z.enum([
+		"untrusted",
+		"on-request",
+		"never"
+	]),
+	dangerouslyBypassApprovalsAndSandbox: z.boolean(),
+	dangerouslyBypassHookTrust: z.boolean(),
+	ephemeral: z.boolean(),
+	ignoreUserConfig: z.boolean(),
+	skipGitRepoCheck: z.boolean(),
+	outputSchema: z.string(),
+	outputLastMessage: z.string(),
+	captureLastMessage: z.boolean(),
+	isolateConfig: z.boolean()
+}).partial();
 /** Generic + nested adapter config for one layer (defaultConfig, case, cell). */
 const ConfigPartialSchema = z.object({
 	model: z.string(),
 	cwd: z.string(),
 	timeoutMs: z.number().int().positive(),
 	env: z.record(z.string(), z.string()),
-	claudeCode: ClaudeCodeConfigSchema
+	claudeCode: ClaudeCodeConfigSchema,
+	codex: CodexConfigSchema
 }).partial();
 /** A matrix cell — one point in the configuration matrix. */
 const MatrixCellSchema = z.object({
@@ -763,84 +716,260 @@ function typeOf(x) {
 	return typeof x;
 }
 //#endregion
-//#region src/config/grading-schema.ts
+//#region src/config/loader-internals.ts
 /**
-* Zod schema for standalone grading YAML (`grading.yaml`).
-*
-* The top-level `judge` block reuses {@link ConfigPartialSchema} fields plus
-* grader-specific concurrency and system-instruction overrides.
+* Shared suite loader helpers (case file collection and parsing).
 */
-/** Top-level `judge` block — mirrors harness config fields plus grader concurrency. */
-const JudgeConfigSchema = ConfigPartialSchema.extend({
-	adapter: z.string().optional(),
-	maxConcurrent: z.number().int().positive().optional(),
-	/** Optional judge prompt prefix (maps to upstream system_instruction). */
-	system_instruction: z.string().optional()
-});
-const GradingConfigSchema = z.object({ judge: JudgeConfigSchema });
-//#endregion
-//#region src/config/grading-loader.ts
-/**
-* Load standalone grading YAML for `harness-eval grade`.
-*
-* Grading config defines the judge subprocess (model, concurrency, Claude Code
-* flags) separately from the suite under test.
-*/
-/** Load grading YAML from disk and resolve relative paths. */
-async function loadGradingConfig(filePath) {
-	const absolutePath = resolve(filePath);
-	let content;
-	try {
-		content = await readFile(absolutePath, "utf8");
-	} catch (err) {
-		throw new ConfigError(`failed to read grading config: ${err instanceof Error ? err.message : String(err)}`, filePath);
-	}
-	return parseGradingConfig(content, absolutePath);
-}
-/**
-* Parse grading YAML from a string.
-*
-* @param sourcePath Optional path for error messages and path resolution.
-*/
-function parseGradingConfig(yamlContent, sourcePath) {
+/** Parse one case file: single case, array, or `{ cases: [...] }`. */
+function parseCasesFile(yamlContent, sourcePath) {
 	let raw;
 	try {
 		raw = parse(yamlContent);
 	} catch (err) {
 		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
 	}
-	const validated = GradingConfigSchema.safeParse(raw);
-	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError$1(validated.error, sourcePath)}`, sourcePath);
-	const config = { judge: { ...validated.data.judge } };
-	if (sourcePath) resolveGradingConfigPaths(config, sourcePath);
-	return config;
+	return transformTestCases(extractRawCases(raw, sourcePath), sourcePath ?? "cases");
 }
-/** Format a zod validation error with optional source file prefix. */
-function formatZodError$1(err, sourcePath) {
+function extractRawCases(raw, sourcePath) {
+	if (Array.isArray(raw)) return raw.map((item, index) => validateRawCase(item, sourcePath, index));
+	if (raw && typeof raw === "object") {
+		const obj = raw;
+		if (Array.isArray(obj.cases)) return obj.cases.map((item, index) => validateRawCase(item, sourcePath, index));
+		if ("id" in obj && "prompt" in obj && "assertions" in obj) return [validateRawCase(raw, sourcePath, 0)];
+	}
+	throw new ConfigError("expected a case object, array of cases, or { cases: [...] }", sourcePath);
+}
+function validateRawCase(raw, sourcePath, index) {
+	const validated = TestCaseSchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError$3(validated.error, sourcePath)}`, sourcePath);
+	return validated.data;
+}
+/** Recursively collect `.yaml` / `.yml` files under `casesDir`. */
+async function collectCaseYamlFiles(casesDir) {
+	const files = [];
+	async function walk(dir) {
+		let entries;
+		try {
+			entries = await readdir(dir, { withFileTypes: true });
+		} catch (err) {
+			if (err instanceof Error && "code" in err && err.code === "ENOENT") return;
+			throw err;
+		}
+		for (const entry of entries) {
+			const fullPath = join(dir, entry.name);
+			if (entry.isDirectory()) await walk(fullPath);
+			else if (entry.isFile() && (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml"))) files.push(fullPath);
+		}
+	}
+	await walk(casesDir);
+	return files.sort();
+}
+function formatZodError$3(err, sourcePath) {
 	return err.issues.map((issue) => {
 		const path = issue.path.length > 0 ? issue.path.join(".") : "(root)";
 		return `  ${sourcePath ? `${sourcePath} → ${path}` : path}: ${issue.message}`;
 	}).join("\n");
 }
 //#endregion
-//#region src/config/loader.ts
+//#region src/config/pipeline-schema.ts
 /**
-* Load a `TestSuite` from a YAML file, directory, or string.
+* Zod schemas for optional `pipeline:` block in suite.yaml.
 *
-* Supports two on-disk layouts:
-*   - Single file: `suite.yaml` with inline `cases`.
-*   - Directory: `suite.yaml` plus optional `cases/*.yaml` fragments merged
-*     in lexicographic path order.
+* Step presence under `pipeline` enables orchestration via `harness-eval pipeline`.
+*/
+/** `pipeline.run` step — harness eval run. */
+const PipelineRunStepSchema = z.object({
+	output: z.string().min(1).optional(),
+	maxConcurrent: z.number().int().positive().optional()
+}).optional();
+/** `pipeline.grade` step — LLM outcome grading. */
+const PipelineGradeStepSchema = z.object({
+	input: z.string().min(1).optional(),
+	output: z.string().min(1).optional(),
+	maxConcurrent: z.number().int().positive().optional()
+}).optional();
+/** `pipeline.envelope` step — EvalRunEnvelope export. */
+const PipelineEnvelopeStepSchema = z.object({
+	report: z.string().min(1).optional(),
+	grading: z.string().min(1).optional(),
+	output: z.string().min(1).optional(),
+	projection: z.enum([
+		"envelope",
+		"trajectory",
+		"instances"
+	]).optional(),
+	includeRawStreamEvents: z.boolean().optional(),
+	noTranscript: z.boolean().optional()
+}).optional();
+/** Top-level optional pipeline block in suite.yaml. */
+const PipelineConfigSchema = z.object({
+	run: PipelineRunStepSchema,
+	grade: PipelineGradeStepSchema,
+	envelope: PipelineEnvelopeStepSchema
+}).partial();
+/** Default artifact filenames relative to the suite.yaml directory. */
+const DEFAULT_PIPELINE_OUTPUTS = {
+	run: "report.json",
+	grade: "grading.json",
+	envelope: "envelope.json"
+};
+//#endregion
+//#region src/config/paths.ts
+/**
+* Resolve relative paths in suite config against the suite file directory.
 *
-* Relative paths in config (MCP config, plugin dirs, etc.) are resolved
-* against the suite file directory after load.
+* YAML authors write paths relative to the suite file; this module absolutizes
+* them at load time so the runner and adapters receive filesystem-ready values.
+* Tilde-prefixed paths and inline JSON blobs (settings starting with `{`) are
+* left unchanged.
 */
+/** Resolve a single path relative to `suiteDir` unless already absolute or `~/`. */
+function resolvePath(value, suiteDir) {
+	if (isAbsolute(value) || value.startsWith("~/")) return value;
+	return join(suiteDir, value);
+}
+/** Resolve Claude Code-specific path fields within a config block. */
+function resolveClaudeCodePaths(block, suiteDir) {
+	const resolved = { ...block };
+	if (typeof resolved.mcpConfig === "string") resolved.mcpConfig = resolvePath(resolved.mcpConfig, suiteDir);
+	if (Array.isArray(resolved.pluginDirs)) resolved.pluginDirs = resolved.pluginDirs.map((p) => typeof p === "string" ? resolvePath(p, suiteDir) : p);
+	if (Array.isArray(resolved.addDirs)) resolved.addDirs = resolved.addDirs.map((p) => typeof p === "string" ? resolvePath(p, suiteDir) : p);
+	for (const field of [
+		"systemPromptFile",
+		"appendSystemPromptFile",
+		"debugFile"
+	]) {
+		const value = resolved[field];
+		if (typeof value === "string" && !value.trim().startsWith("{")) resolved[field] = resolvePath(value, suiteDir);
+	}
+	if (typeof resolved.settings === "string" && !resolved.settings.trim().startsWith("{")) resolved.settings = resolvePath(resolved.settings, suiteDir);
+	return resolved;
+}
+/** Resolve Codex-specific path fields within a config block. */
+function resolveCodexPaths(block, suiteDir) {
+	const resolved = { ...block };
+	if (Array.isArray(resolved.addDirs)) resolved.addDirs = resolved.addDirs.map((p) => typeof p === "string" ? resolvePath(p, suiteDir) : p);
+	for (const field of ["outputSchema", "outputLastMessage"]) {
+		const value = resolved[field];
+		if (typeof value === "string") resolved[field] = resolvePath(value, suiteDir);
+	}
+	return resolved;
+}
+/** Resolve relative paths in a config layer relative to `suiteDir`. */
+function resolveConfigPaths(config, suiteDir) {
+	if (!config) return void 0;
+	const resolved = { ...config };
+	if (typeof resolved.cwd === "string") resolved.cwd = resolvePath(resolved.cwd, suiteDir);
+	if (resolved.claudeCode && typeof resolved.claudeCode === "object" && !Array.isArray(resolved.claudeCode)) resolved.claudeCode = resolveClaudeCodePaths(resolved.claudeCode, suiteDir);
+	if (resolved.codex && typeof resolved.codex === "object" && !Array.isArray(resolved.codex)) resolved.codex = resolveCodexPaths(resolved.codex, suiteDir);
+	return resolved;
+}
+/** Resolve paths on an entire suite after load. */
+function resolveSuitePaths(suite, suiteFilePath) {
+	const suiteDir = configFileDir(suiteFilePath);
+	suite.defaultConfig = resolveConfigPaths(suite.defaultConfig, suiteDir);
+	for (const cell of suite.matrix) cell.config = resolveConfigPaths(cell.config, suiteDir) ?? cell.config;
+	for (const testCase of suite.cases) testCase.config = resolveConfigPaths(testCase.config, suiteDir);
+}
+/** Parent directory of a suite or grading config file path. */
+function configFileDir(filePath) {
+	return filePath.includes("/") || filePath.includes("\\") ? filePath.replace(/[/\\][^/\\]+$/, "") : ".";
+}
 /**
-* Load a suite from a file path or directory path.
+* Heuristically resolve env var values that look like relative file paths.
 *
-* @throws {@link ConfigError} when the path is unreadable or validation fails.
+* Used for grading config where credential or config paths may be expressed
+* relative to the grading YAML location.
 */
-async function loadSuite(filePath) {
+function resolveEnvPaths(env, baseDir) {
+	const resolved = {};
+	for (const [key, value] of Object.entries(env)) if (value.startsWith("./") || value.startsWith("../")) resolved[key] = resolvePath(value, baseDir);
+	else resolved[key] = value;
+	return resolved;
+}
+/** Resolve relative paths in a standalone grading config file. */
+function resolveGradingConfigPaths(config, configFilePath) {
+	const baseDir = configFileDir(configFilePath);
+	const { adapter, maxConcurrent, ...rest } = config.judge;
+	config.judge = {
+		...resolveConfigPaths(rest, baseDir) ?? rest,
+		adapter,
+		maxConcurrent
+	};
+	if (config.judge.env) config.judge.env = resolveEnvPaths(config.judge.env, baseDir);
+}
+/** Resolve a pipeline artifact path relative to the suite.yaml directory. */
+function resolvePipelinePath(value, defaultRelative, suiteDir) {
+	return resolvePath(value ?? defaultRelative, suiteDir);
+}
+/** Resolve relative paths in a parsed pipeline config. */
+function resolvePipelineConfigPaths(pipeline, suiteFilePath) {
+	const suiteDir = configFileDir(suiteFilePath);
+	const resolved = {};
+	if (pipeline.run) resolved.run = resolvePipelineRunStep(pipeline.run, suiteDir);
+	if (pipeline.grade) resolved.grade = resolvePipelineGradeStep(pipeline.grade, suiteDir);
+	if (pipeline.envelope) resolved.envelope = resolvePipelineEnvelopeStep(pipeline.envelope, suiteDir);
+	return resolved;
+}
+/** Resolve one pipeline step's run output path. */
+function resolvePipelineRunStep(step, suiteDir) {
+	return {
+		...step,
+		output: resolvePipelinePath(step.output, DEFAULT_PIPELINE_OUTPUTS.run, suiteDir)
+	};
+}
+/** Resolve grade step input (optional) and output paths. */
+function resolvePipelineGradeStep(step, suiteDir) {
+	return {
+		...step,
+		input: step.input ? resolvePipelinePath(step.input, DEFAULT_PIPELINE_OUTPUTS.run, suiteDir) : void 0,
+		output: resolvePipelinePath(step.output, DEFAULT_PIPELINE_OUTPUTS.grade, suiteDir)
+	};
+}
+/** Resolve envelope step report, grading, and output paths. */
+function resolvePipelineEnvelopeStep(step, suiteDir) {
+	return {
+		...step,
+		report: step.report ? resolvePipelinePath(step.report, DEFAULT_PIPELINE_OUTPUTS.run, suiteDir) : void 0,
+		grading: step.grading ? resolvePipelinePath(step.grading, DEFAULT_PIPELINE_OUTPUTS.grade, suiteDir) : void 0,
+		output: resolvePipelinePath(step.output, DEFAULT_PIPELINE_OUTPUTS.envelope, suiteDir)
+	};
+}
+//#endregion
+//#region src/config/grading-schema.ts
+/**
+* Zod schema for standalone grading YAML (`grading.yaml`).
+*
+* The top-level `judge` block reuses {@link ConfigPartialSchema} fields plus
+* grader-specific concurrency and system-instruction overrides.
+*/
+/** Top-level `judge` block — mirrors harness config fields plus grader concurrency. */
+const JudgeConfigSchema = ConfigPartialSchema.extend({
+	adapter: z.string().optional(),
+	maxConcurrent: z.number().int().positive().optional(),
+	/** Optional judge prompt prefix (maps to upstream system_instruction). */
+	system_instruction: z.string().optional()
+});
+const GradingConfigSchema = z.object({ judge: JudgeConfigSchema });
+//#endregion
+//#region src/config/suite-file-schema.ts
+/** Single-file suite with optional inline judge and pipeline orchestration. */
+const SuiteFileSingleSchema = TestSuiteSchema.extend({
+	judge: JudgeConfigSchema.optional(),
+	pipeline: PipelineConfigSchema.optional()
+});
+/** Directory suite root with optional inline judge and pipeline orchestration. */
+const SuiteFileDirectorySchema = SuiteDirectorySchema.extend({
+	judge: JudgeConfigSchema.optional(),
+	pipeline: PipelineConfigSchema.optional()
+});
+//#endregion
+//#region src/config/suite-document-loader.ts
+/**
+* Load a unified suite.yaml document (suite + optional judge + pipeline).
+*/
+/** Load suite.yaml (or directory) including optional judge and pipeline blocks. */
+async function loadSuiteDocument(filePath, options = {}) {
 	const absolutePath = resolve(filePath);
 	let info;
 	try {
@@ -848,26 +977,12 @@ async function loadSuite(filePath) {
 	} catch (err) {
 		throw new ConfigError(`failed to read suite path: ${err instanceof Error ? err.message : String(err)}`, filePath);
 	}
-	if (info.isDirectory()) return loadSuiteDirectory(absolutePath);
-	return loadSuiteFile(absolutePath);
+	const strict = options.validateOrchestration !== false;
+	if (info.isDirectory()) return loadSuiteDocumentDirectory(absolutePath, strict);
+	return loadSuiteDocumentFile(absolutePath, strict);
 }
-/** Load and parse a single-file suite (not a directory layout). */
-async function loadSuiteFile(absolutePath) {
-	let content;
-	try {
-		content = await readFile(absolutePath, "utf8");
-	} catch (err) {
-		throw new ConfigError(`failed to read suite file: ${err instanceof Error ? err.message : String(err)}`, absolutePath);
-	}
-	return parseSuite(content, absolutePath);
-}
-/**
-* Load a directory suite: `suite.yaml` plus optional `cases/` YAML files.
-*
-* Cases from `suite.yaml` sort before external case files; within each file,
-* array order is preserved.
-*/
-async function loadSuiteDirectory(dir) {
+/** Load suite.yaml from a directory layout (cases under `cases/`). */
+async function loadSuiteDocumentDirectory(dir, strict) {
 	const suiteYamlPath = join(dir, "suite.yaml");
 	let content;
 	try {
@@ -875,7 +990,7 @@ async function loadSuiteDirectory(dir) {
 	} catch (err) {
 		throw new ConfigError(`missing suite.yaml in suite directory: ${err instanceof Error ? err.message : String(err)}`, dir);
 	}
-	const base = parseSuiteDirectory(content, suiteYamlPath);
+	const { judge, pipeline, suite: base } = parseSuiteFileRoot(content, suiteYamlPath, "directory", strict);
 	const casesDir = join(dir, "cases");
 	const caseFiles = await collectCaseYamlFiles(casesDir);
 	const tagged = base.cases.map((testCase, index) => ({
@@ -904,94 +1019,242 @@ async function loadSuiteDirectory(dir) {
 		cases
 	};
 	resolveSuitePaths(suite, suiteYamlPath);
-	return suite;
+	return buildSuiteDocument(suiteYamlPath, suite, judge, pipeline);
+}
+/** Load a single suite.yaml file (inline cases). */
+async function loadSuiteDocumentFile(absolutePath, strict) {
+	let content;
+	try {
+		content = await readFile(absolutePath, "utf8");
+	} catch (err) {
+		throw new ConfigError(`failed to read suite file: ${err instanceof Error ? err.message : String(err)}`, absolutePath);
+	}
+	const { judge, pipeline, suite } = parseSuiteFileRoot(content, absolutePath, "single", strict);
+	resolveSuitePaths(suite, absolutePath);
+	return buildSuiteDocument(absolutePath, suite, judge, pipeline);
 }
 /**
-* Parse suite YAML from a string (single-file layout with inline cases).
+* Parse suite.yaml root and validate against the appropriate schema.
 *
-* @param sourcePath Optional path for error messages and relative path resolution.
+* When `strict` is true, uses extended schemas that validate `judge:` and
+* `pipeline:` blocks (for `loadSuiteDocument`). When false, uses base schemas
+* that silently strip unknown keys (for `loadSuite`).
 */
-function parseSuite(yamlContent, sourcePath) {
+function parseSuiteFileRoot(yamlContent, sourcePath, layout, strict) {
 	let raw;
 	try {
 		raw = parse(yamlContent);
 	} catch (err) {
 		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
 	}
-	const validated = TestSuiteSchema.safeParse(raw);
-	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError(validated.error, sourcePath)}`, sourcePath);
-	const suite = transformSuite(validated.data);
-	if (sourcePath) resolveSuitePaths(suite, resolve(sourcePath));
-	return suite;
+	if (!strict) {
+		const validated = (layout === "directory" ? SuiteDirectorySchema : TestSuiteSchema).safeParse(raw);
+		if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError$2(validated.error, sourcePath)}`, sourcePath);
+		return { suite: (layout === "directory" ? transformSuiteDirectory : transformSuite)(validated.data) };
+	}
+	if (layout === "directory") {
+		const validated = SuiteFileDirectorySchema.safeParse(raw);
+		if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError$2(validated.error, sourcePath)}`, sourcePath);
+		return extractSuiteFileParts(validated.data, sourcePath, transformSuiteDirectory);
+	}
+	const validated = SuiteFileSingleSchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError$2(validated.error, sourcePath)}`, sourcePath);
+	return extractSuiteFileParts(validated.data, sourcePath, transformSuite);
 }
-/** Parse `suite.yaml` for directory layout (cases may be omitted). */
-function parseSuiteDirectory(yamlContent, sourcePath) {
+/** Split validated YAML into suite, judge, and pipeline with path resolution. */
+function extractSuiteFileParts(data, sourcePath, transform) {
+	const { judge: rawJudge, pipeline: rawPipeline, ...suiteRaw } = data;
+	const suite = transform(suiteRaw);
+	let judge;
+	if (rawJudge) {
+		judge = { ...rawJudge };
+		resolveGradingConfigPaths({ judge }, sourcePath);
+	}
+	let pipeline;
+	if (rawPipeline) {
+		pipeline = transformPipelineConfig(rawPipeline);
+		pipeline = resolvePipelineConfigPaths(pipeline, sourcePath);
+	}
+	return {
+		suite,
+		judge,
+		pipeline
+	};
+}
+/** Apply default artifact filenames when a pipeline step key is present but paths are omitted. */
+function transformPipelineConfig(raw) {
+	const pipeline = {};
+	if (raw.run !== void 0) pipeline.run = {
+		output: raw.run?.output ?? DEFAULT_PIPELINE_OUTPUTS.run,
+		maxConcurrent: raw.run?.maxConcurrent
+	};
+	if (raw.grade !== void 0) pipeline.grade = {
+		input: raw.grade?.input,
+		output: raw.grade?.output ?? DEFAULT_PIPELINE_OUTPUTS.grade,
+		maxConcurrent: raw.grade?.maxConcurrent
+	};
+	if (raw.envelope !== void 0) pipeline.envelope = {
+		report: raw.envelope?.report,
+		grading: raw.envelope?.grading,
+		output: raw.envelope?.output ?? DEFAULT_PIPELINE_OUTPUTS.envelope,
+		projection: raw.envelope?.projection ?? "envelope",
+		includeRawStreamEvents: raw.envelope?.includeRawStreamEvents,
+		noTranscript: raw.envelope?.noTranscript
+	};
+	return pipeline;
+}
+/** Assemble the runtime {@link SuiteDocument} from parsed parts. */
+function buildSuiteDocument(suitePath, suite, judge, pipeline) {
+	return {
+		suitePath: resolve(suitePath),
+		suite,
+		judge,
+		pipeline
+	};
+}
+function formatZodError$2(err, sourcePath) {
+	return err.issues.map((issue) => {
+		const path = issue.path.length > 0 ? issue.path.join(".") : "(root)";
+		return `  ${sourcePath ? `${sourcePath} → ${path}` : path}: ${issue.message}`;
+	}).join("\n");
+}
+//#endregion
+//#region src/config/grading-loader.ts
+/**
+* Load standalone grading YAML for `harness-eval grade`.
+*
+* Also accepts unified suite.yaml files with an inline `judge:` block.
+*/
+/** Load grading YAML from disk and resolve relative paths. */
+async function loadGradingConfig(filePath) {
+	const absolutePath = resolve(filePath);
+	let info;
+	try {
+		info = await stat(absolutePath);
+	} catch (err) {
+		throw new ConfigError(`failed to read grading config: ${err instanceof Error ? err.message : String(err)}`, filePath);
+	}
+	if (info.isDirectory()) return loadGradingFromSuiteYaml(join(absolutePath, "suite.yaml"));
+	let content;
+	try {
+		content = await readFile(absolutePath, "utf8");
+	} catch (err) {
+		throw new ConfigError(`failed to read grading config: ${err instanceof Error ? err.message : String(err)}`, filePath);
+	}
+	if (isSuiteRoot(parse(content))) return parseGradingFromSuiteRaw(parse(content), absolutePath);
+	return parseGradingConfig(content, absolutePath);
+}
+/**
+* Parse grading YAML from a string.
+*
+* @param sourcePath Optional path for error messages and path resolution.
+*/
+function parseGradingConfig(yamlContent, sourcePath) {
 	let raw;
 	try {
 		raw = parse(yamlContent);
 	} catch (err) {
 		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
 	}
-	const validated = SuiteDirectorySchema.safeParse(raw);
-	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError(validated.error, sourcePath)}`, sourcePath);
-	return transformSuiteDirectory(validated.data);
+	if (isSuiteRoot(raw)) return parseGradingFromSuiteRaw(raw, sourcePath ?? "suite.yaml");
+	const validated = GradingConfigSchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError$1(validated.error, sourcePath)}`, sourcePath);
+	const config = { judge: { ...validated.data.judge } };
+	if (sourcePath) resolveGradingConfigPaths(config, sourcePath);
+	return config;
 }
-/** Parse one case file: single case, array, or `{ cases: [...] }`. */
-function parseCasesFile(yamlContent, sourcePath) {
+/** Detect unified suite.yaml by presence of suite-specific keys (vs standalone grading YAML). */
+function isSuiteRoot(raw) {
+	if (raw === null || typeof raw !== "object") return false;
+	return "cases" in raw || "matrix" in raw && "adapter" in raw;
+}
+async function loadGradingFromSuiteYaml(suiteYamlPath) {
+	let content;
+	try {
+		content = await readFile(suiteYamlPath, "utf8");
+	} catch (err) {
+		throw new ConfigError(`failed to read suite file: ${err instanceof Error ? err.message : String(err)}`, suiteYamlPath);
+	}
 	let raw;
 	try {
-		raw = parse(yamlContent);
+		raw = parse(content);
 	} catch (err) {
-		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
+		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, suiteYamlPath);
 	}
-	return transformTestCases(extractRawCases(raw, sourcePath), sourcePath ?? "cases");
+	return parseGradingFromSuiteRaw(raw, suiteYamlPath);
 }
+function parseGradingFromSuiteRaw(raw, sourcePath) {
+	const single = SuiteFileSingleSchema.safeParse(raw);
+	if (single.success) {
+		if (!single.data.judge) throw new ConfigError("suite file has no judge block", sourcePath);
+		const config = { judge: { ...single.data.judge } };
+		resolveGradingConfigPaths(config, sourcePath);
+		return config;
+	}
+	const directory = SuiteFileDirectorySchema.safeParse(raw);
+	if (directory.success) {
+		if (!directory.data.judge) throw new ConfigError("suite file has no judge block", sourcePath);
+		const config = { judge: { ...directory.data.judge } };
+		resolveGradingConfigPaths(config, sourcePath);
+		return config;
+	}
+	throw new ConfigError(`validation failed:\n${formatZodError$1(directory.error ?? single.error, sourcePath)}`, sourcePath);
+}
+/** Format a zod validation error with optional source file prefix. */
+function formatZodError$1(err, sourcePath) {
+	return err.issues.map((issue) => {
+		const path = issue.path.length > 0 ? issue.path.join(".") : "(root)";
+		return `  ${sourcePath ? `${sourcePath} → ${path}` : path}: ${issue.message}`;
+	}).join("\n");
+}
+//#endregion
+//#region src/config/loader.ts
 /**
-* Normalize raw YAML into a list of {@link RawTestCase} objects.
+* Load a `TestSuite` from a YAML file, directory, or string.
 *
-* Accepts a single case, an array, or `{ cases: [...] }`.
+* For unified suite.yaml with optional `judge:` and `pipeline:` blocks,
+* use {@link loadSuiteDocument}.
 */
-function extractRawCases(raw, sourcePath) {
-	if (Array.isArray(raw)) return raw.map((item, index) => validateRawCase(item, sourcePath, index));
-	if (raw && typeof raw === "object") {
-		const obj = raw;
-		if (Array.isArray(obj.cases)) return obj.cases.map((item, index) => validateRawCase(item, sourcePath, index));
-		if ("id" in obj && "prompt" in obj && "assertions" in obj) return [validateRawCase(raw, sourcePath, 0)];
-	}
-	throw new ConfigError("expected a case object, array of cases, or { cases: [...] }", sourcePath);
-}
-/** Validate one raw case object against {@link TestCaseSchema}. */
-function validateRawCase(raw, sourcePath, index) {
-	const validated = TestCaseSchema.safeParse(raw);
-	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError(validated.error, sourcePath)}`, sourcePath);
-	return validated.data;
+/**
+* Load a suite from a file path or directory path (suite portion only).
+*
+* Orchestration blocks (`judge:`, `pipeline:`) are silently stripped — callers
+* that only need the `TestSuite` are not broken by malformed orchestration YAML.
+* Use {@link loadSuiteDocument} when you need validated orchestration metadata.
+*/
+async function loadSuite(filePath) {
+	return (await loadSuiteDocument(filePath, { validateOrchestration: false })).suite;
 }
 /**
-* Recursively collect `.yaml` / `.yml` files under `casesDir`.
+* Parse suite YAML from a string (single-file layout with inline cases).
 *
-* Returns an empty list when the directory does not exist — external cases
-* are optional in directory layout.
+* Unknown top-level keys such as `judge` and `pipeline` are stripped.
 */
-async function collectCaseYamlFiles(casesDir) {
-	const files = [];
-	async function walk(dir) {
-		let entries;
-		try {
-			entries = await readdir(dir, { withFileTypes: true });
-		} catch (err) {
-			if (err instanceof Error && "code" in err && err.code === "ENOENT") return;
-			throw err;
-		}
-		for (const entry of entries) {
-			const fullPath = join(dir, entry.name);
-			if (entry.isDirectory()) await walk(fullPath);
-			else if (entry.isFile() && (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml"))) files.push(fullPath);
-		}
+function parseSuite(yamlContent, sourcePath) {
+	let raw;
+	try {
+		raw = parse(yamlContent);
+	} catch (err) {
+		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
 	}
-	await walk(casesDir);
-	return files.sort();
+	const validated = TestSuiteSchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError(validated.error, sourcePath)}`, sourcePath);
+	const suite = transformSuite(validated.data);
+	if (sourcePath) resolveSuitePaths(suite, resolve(sourcePath));
+	return suite;
+}
+/** Parse `suite.yaml` for directory layout (cases may be omitted). @internal */
+function parseSuiteDirectory(yamlContent, sourcePath) {
+	let raw;
+	try {
+		raw = parse(yamlContent);
+	} catch (err) {
+		throw new ConfigError(`YAML parse error: ${err instanceof Error ? err.message : String(err)}`, sourcePath);
+	}
+	const validated = SuiteDirectorySchema.safeParse(raw);
+	if (!validated.success) throw new ConfigError(`validation failed:\n${formatZodError(validated.error, sourcePath)}`, sourcePath);
+	return transformSuiteDirectory(validated.data);
 }
-/** Format a zod validation error with optional source file prefix. */
 function formatZodError(err, sourcePath) {
 	return err.issues.map((issue) => {
 		const path = issue.path.length > 0 ? issue.path.join(".") : "(root)";
@@ -999,6 +1262,6 @@ function formatZodError(err, sourcePath) {
 	}).join("\n");
 }
 //#endregion
-export { parseGradingConfig as a, loadGradingConfig as i, parseCasesFile as n, ConfigError as o, parseSuite as r, loadSuite as t };
+export { parseGradingConfig as a, parseCasesFile as c, loadGradingConfig as i, ConfigError as l, parseSuite as n, loadSuiteDocument as o, parseSuiteDirectory as r, DEFAULT_PIPELINE_OUTPUTS as s, loadSuite as t };
 
-//# sourceMappingURL=loader-DcI0KfRX.js.map
+//# sourceMappingURL=loader-DnQ6Jt0i.js.map