@alis-build/harness-eval 0.1.1 → 0.1.3

package/dist/reporter-Biy-5-9M.js

@@ -0,0 +1,2216 @@
+import { i as buildJudgeArgs } from "./claude-code-C_7hxC8z.js";
+import { n as createLimit, t as runSuite, u as getAdapter } from "./suite-BcP64nlb.js";
+import { s as buildJudgeArgs$1 } from "./codex-0cHO2te9.js";
+import { i as loadGradingConfig, l as ConfigError, o as loadSuiteDocument, s as DEFAULT_PIPELINE_OUTPUTS, t as loadSuite } from "./loader-DnQ6Jt0i.js";
+import { spawn } from "node:child_process";
+import { readFile, stat, writeFile } from "node:fs/promises";
+import { basename, dirname, join, resolve } from "node:path";
+import { createHash, randomUUID } from "node:crypto";
+import { parse } from "yaml";
+import { fileURLToPath } from "node:url";
+//#region src/types/eval-record.ts
+/** Schema version for {@link EvalRunEnvelope} JSON documents. */
+const EVAL_RUN_SCHEMA_VERSION = "1.0";
+/** Schema version embedded in each {@link TrajectoryView} at export time. */
+const TRAJECTORY_SCHEMA_VERSION = "1.0";
+//#endregion
+//#region src/grader/prompt.ts
+/**
+* Build the full grader prompt including eval prompt, transcript, and schema.
+*
+* When `systemInstruction` is set it is prepended as a judge-specific prefix.
+*/
+function buildGraderPrompt(input) {
+	const expectationList = input.expectations.map((e, i) => `${i + 1}. ${e}`).join("\n");
+	return `${input.systemInstruction ? `${input.systemInstruction.trim()}\n\n` : ""}You are an automated evaluation grader (not the agent under test). Your only job is to score expectations against the transcript below.
+
+Your job is to evaluate each expectation against the transcript and final response.
+PASS only when there is clear evidence in the transcript or final response.
+When uncertain, FAIL — burden of proof is on PASS.
+
+Also critique the expectations themselves if any are trivially satisfied or miss important outcomes.
+
+## Eval prompt
+
+${input.prompt}
+
+## Execution transcript
+
+${input.transcript}
+
+## Expectations to grade
+
+${expectationList}
+
+## Output format
+
+Respond with ONLY a single JSON object (no markdown fences, no commentary) matching this schema:
+
+{
+  "expectations": [
+    { "text": "<original expectation>", "passed": true|false, "evidence": "<quote or description>" }
+  ],
+  "summary": { "passed": <int>, "failed": <int>, "total": <int>, "pass_rate": <0.0-1.0> },
+  "eval_feedback": {
+    "suggestions": [{ "assertion": "<optional>", "reason": "<string>" }],
+    "overall": "<brief assessment>"
+  }
+}
+
+Include every expectation in the same order. summary must match the expectations array.`;
+}
+//#endregion
+//#region src/grader/parse.ts
+/**
+* Extract assistant text from Claude stdout.
+*
+* Handles plain text, single JSON result envelopes, stream-json arrays, and
+* assistant message objects — the judge subprocess may emit any of these
+* depending on Claude Code version and flags.
+*/
+function extractClaudeResponseText(stdout) {
+	const trimmed = stdout.trim();
+	if (!trimmed) return "";
+	try {
+		const data = JSON.parse(trimmed);
+		if (Array.isArray(data)) return extractFromEventArray(data) ?? trimmed;
+		if (typeof data === "object" && data !== null) {
+			const event = data;
+			if (event.type === "result" && typeof event.result === "string") return event.result;
+			if (event.type === "assistant" && event.message) {
+				const text = textFromAssistantMessage(event.message);
+				if (text) return text;
+			}
+		}
+	} catch {}
+	return trimmed;
+}
+/**
+* Extract assistant text from Codex judge stdout.
+*
+* Handles plain text and JSONL streams from accidental `--json` usage.
+*/
+function extractCodexResponseText(stdout) {
+	const trimmed = stdout.trim();
+	if (!trimmed) return "";
+	const lines = trimmed.split("\n").filter((line) => line.trim().length > 0);
+	if (lines.length > 1) for (let i = lines.length - 1; i >= 0; i--) try {
+		const event = JSON.parse(lines[i]);
+		if (event.type === "item.completed" && (event.item?.type === "assistant_message" || event.item?.item_type === "assistant_message") && event.item.text) return event.item.text;
+	} catch {
+		continue;
+	}
+	return trimmed;
+}
+/** Walk a stream-json event array and return the final assistant or result text. */
+function extractFromEventArray(events) {
+	const result = events.find((e) => typeof e === "object" && e !== null && e.type === "result");
+	if (result?.result) return result.result;
+	const assistantTexts = [];
+	for (const event of events) if (typeof event === "object" && event !== null && event.type === "assistant") {
+		const text = textFromAssistantMessage(event.message);
+		if (text) assistantTexts.push(text);
+	}
+	if (assistantTexts.length > 0) return assistantTexts[assistantTexts.length - 1];
+	return null;
+}
+/** Concatenate text blocks from an Anthropic-style assistant message object. */
+function textFromAssistantMessage(message) {
+	if (!message || typeof message !== "object") return null;
+	const content = message.content;
+	if (typeof content === "string") return content;
+	if (!Array.isArray(content)) return null;
+	const texts = [];
+	for (const block of content) if (typeof block === "object" && block !== null && block.type === "text" && typeof block.text === "string") texts.push(block.text);
+	return texts.length > 0 ? texts.join("\n") : null;
+}
+/**
+* Parse grader JSON from response text.
+*
+* Tries the raw string first, then fenced code blocks and brace-delimited
+* substrings. Returns null when no valid expectations array is found.
+*/
+function parseGraderJson(text) {
+	const candidates = [text.trim(), extractJsonBlock(text)];
+	for (const candidate of candidates) {
+		if (!candidate) continue;
+		try {
+			const normalized = normalizeGraderJson(JSON.parse(candidate));
+			if (normalized.expectations.length > 0) return normalized;
+		} catch {
+			continue;
+		}
+	}
+	return null;
+}
+/** Extract JSON from markdown fences or the outermost `{...}` substring. */
+function extractJsonBlock(text) {
+	const fence = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+	if (fence?.[1]) return fence[1].trim();
+	const start = text.indexOf("{");
+	const end = text.lastIndexOf("}");
+	if (start >= 0 && end > start) return text.slice(start, end + 1);
+	return null;
+}
+/** Map raw grader JSON to runtime {@link GraderOutput} with computed summary. */
+function normalizeGraderJson(raw) {
+	const expectations = (raw.expectations ?? []).map((e) => ({
+		text: e.text ?? "",
+		passed: Boolean(e.passed),
+		evidence: e.evidence ?? ""
+	}));
+	const passed = expectations.filter((e) => e.passed).length;
+	const failed = expectations.length - passed;
+	const total = expectations.length;
+	const passRate = raw.summary?.pass_rate ?? raw.summary?.passRate ?? (total === 0 ? 0 : passed / total);
+	const summary = {
+		passed: raw.summary?.passed ?? passed,
+		failed: raw.summary?.failed ?? failed,
+		total: raw.summary?.total ?? total,
+		passRate
+	};
+	let evalFeedback;
+	if (raw.eval_feedback) evalFeedback = {
+		suggestions: (raw.eval_feedback.suggestions ?? []).map((s) => ({
+			assertion: s.assertion,
+			reason: s.reason ?? ""
+		})),
+		overall: raw.eval_feedback.overall ?? ""
+	};
+	return {
+		expectations,
+		summary,
+		evalFeedback
+	};
+}
+//#endregion
+//#region src/grader/spawn-judge.ts
+/**
+* Shared subprocess utilities for judge graders (Claude + Codex).
+*
+* Owns detached spawn, process-group teardown, and SIGTERM → SIGKILL
+* escalation so both graders share one implementation.
+*/
+const KILL_GRACE_MS = 5e3;
+/** Kill the detached process group (fallback to single process if group kill fails). */
+function killTree(child, signal) {
+	if (child.pid === void 0) return;
+	try {
+		process.kill(-child.pid, signal);
+	} catch {
+		try {
+			child.kill(signal);
+		} catch {}
+	}
+}
+/**
+* Spawn a judge subprocess with process-group teardown and collect stdout.
+*
+* Non-zero exit with empty stdout is treated as failure; partial stdout on
+* non-zero exit is retained (judges sometimes exit non-zero after emitting JSON).
+*/
+function spawnCollectStdout(options) {
+	const { binary, args, timeoutMs, env, cwd } = options;
+	return new Promise((resolve, reject) => {
+		const child = spawn(binary, args, {
+			env: env ?? process.env,
+			cwd,
+			stdio: [
+				"ignore",
+				"pipe",
+				"pipe"
+			],
+			detached: true
+		});
+		const chunks = [];
+		child.stdout?.setEncoding("utf8");
+		child.stdout?.on("data", (c) => chunks.push(c));
+		const stderrChunks = [];
+		child.stderr?.setEncoding("utf8");
+		child.stderr?.on("data", (c) => stderrChunks.push(c));
+		let killEscalation = null;
+		const timer = setTimeout(() => {
+			killTree(child, "SIGTERM");
+			killEscalation = setTimeout(() => killTree(child, "SIGKILL"), KILL_GRACE_MS);
+			const stderrHint = stderrChunks.join("").trim().slice(0, 400);
+			reject(/* @__PURE__ */ new Error(`grader timed out after ${timeoutMs}ms` + (stderrHint ? ` (stderr: ${stderrHint})` : "")));
+		}, timeoutMs);
+		const finalize = (err) => {
+			clearTimeout(timer);
+			if (killEscalation) clearTimeout(killEscalation);
+			if (err) reject(err);
+			else resolve(chunks.join(""));
+		};
+		child.on("error", (err) => finalize(err));
+		child.on("close", (code) => {
+			if (code !== 0 && chunks.length === 0) finalize(/* @__PURE__ */ new Error(`grader exited ${code}: ${stderrChunks.join("").slice(0, 500)}`));
+			else finalize();
+		});
+	});
+}
+//#endregion
+//#region src/grader/claude-grader.ts
+/**
+* Grade expectations by spawning Claude as judge (skill-creator grader pattern).
+*/
+const DEFAULT_TIMEOUT_MS$1 = 3e5;
+/**
+* Judge subprocess defaults — grading is a single-shot JSON response, not an agent session.
+* Without these, Claude Code may load plugins/MCP and loop on tools until timeout.
+*/
+const JUDGE_CLAUDE_DEFAULTS = {
+	maxTurns: 1,
+	bare: true,
+	disableSlashCommands: true,
+	noSessionPersistence: true
+};
+/** Merge user-supplied Claude Code options over judge-safe defaults. */
+function mergeJudgeClaudeOptions(claudeCode) {
+	return {
+		...JUDGE_CLAUDE_DEFAULTS,
+		...claudeCode
+	};
+}
+/** Factory returning a {@link GraderFn} bound to subprocess options. */
+function createClaudeGrader(options = {}) {
+	return (input) => runClaudeGrader(input, options);
+}
+/**
+* Spawn Claude as judge, parse JSON response, align with input expectations.
+*
+* Unparseable output fails all expectations and sets {@link GraderOutput.error}.
+*/
+async function runClaudeGrader(input, options = {}) {
+	const binary = options.binary ?? options.claudeCode?.binary ?? "claude";
+	const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS$1;
+	const prompt = buildGraderPrompt(input);
+	const model = options.model ?? options.claudeCode?.model;
+	const responseText = extractClaudeResponseText(await spawnCollectStdout({
+		binary,
+		args: buildJudgeArgs(prompt, {
+			...mergeJudgeClaudeOptions(options.claudeCode),
+			model
+		}),
+		timeoutMs,
+		env: buildChildEnv(options.env),
+		cwd: options.cwd
+	}));
+	const parsed = parseGraderJson(responseText);
+	if (!parsed) return {
+		expectations: input.expectations.map((text) => ({
+			text,
+			passed: false,
+			evidence: "Grader returned unparseable output"
+		})),
+		summary: {
+			passed: 0,
+			failed: input.expectations.length,
+			total: input.expectations.length,
+			passRate: 0
+		},
+		error: `failed to parse grader JSON from response: ${responseText.slice(0, 200)}`
+	};
+	const expectations = input.expectations.map((text, i) => {
+		const graded = parsed.expectations[i];
+		return {
+			text,
+			passed: graded?.passed ?? false,
+			evidence: graded?.evidence ?? "No evidence returned"
+		};
+	});
+	const passed = expectations.filter((e) => e.passed).length;
+	const total = expectations.length;
+	return {
+		expectations,
+		summary: {
+			passed,
+			failed: total - passed,
+			total,
+			passRate: total === 0 ? 0 : passed / total
+		},
+		evalFeedback: parsed.evalFeedback
+	};
+}
+/**
+* Build subprocess env, stripping CLAUDECODE to avoid nested-session guards.
+*/
+function buildChildEnv(extraEnv) {
+	const env = {
+		...process.env,
+		...extraEnv
+	};
+	delete env.CLAUDECODE;
+	return env;
+}
+//#endregion
+//#region src/grader/codex-grader.ts
+/**
+* Grade expectations by spawning Codex as judge.
+*/
+const DEFAULT_TIMEOUT_MS = 3e5;
+/** Judge subprocess defaults — single-shot grading without persistent sessions. */
+const JUDGE_CODEX_DEFAULTS = {
+	ephemeral: true,
+	ignoreUserConfig: true,
+	skipGitRepoCheck: true
+};
+/** Merge user-supplied Codex options over judge-safe defaults. */
+function mergeJudgeCodexOptions(codex) {
+	return {
+		...JUDGE_CODEX_DEFAULTS,
+		...codex
+	};
+}
+/** Factory returning a {@link GraderFn} bound to subprocess options. */
+function createCodexGrader(options = {}) {
+	return (input) => runCodexGrader(input, options);
+}
+/**
+* Spawn Codex as judge, parse JSON response, align with input expectations.
+*
+* Unparseable output fails all expectations and sets {@link GraderOutput.error}.
+*/
+async function runCodexGrader(input, options = {}) {
+	const binary = options.binary ?? options.codex?.binary ?? "codex";
+	const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+	const prompt = buildGraderPrompt(input);
+	const model = options.model ?? options.codex?.model;
+	const responseText = extractCodexResponseText(await spawnCollectStdout({
+		binary,
+		args: buildJudgeArgs$1(prompt, {
+			...mergeJudgeCodexOptions(options.codex),
+			model,
+			cwd: options.cwd
+		}),
+		timeoutMs,
+		env: {
+			...process.env,
+			...options.env
+		},
+		cwd: options.cwd
+	}));
+	const parsed = parseGraderJson(responseText);
+	if (!parsed) return {
+		expectations: input.expectations.map((text) => ({
+			text,
+			passed: false,
+			evidence: "Grader returned unparseable output"
+		})),
+		summary: {
+			passed: 0,
+			failed: input.expectations.length,
+			total: input.expectations.length,
+			passRate: 0
+		},
+		error: `failed to parse grader JSON from response: ${responseText.slice(0, 200)}`
+	};
+	const expectations = input.expectations.map((text, i) => {
+		const graded = parsed.expectations[i];
+		return {
+			text,
+			passed: graded?.passed ?? false,
+			evidence: graded?.evidence ?? "No evidence returned"
+		};
+	});
+	const passed = expectations.filter((e) => e.passed).length;
+	const total = expectations.length;
+	return {
+		expectations,
+		summary: {
+			passed,
+			failed: total - passed,
+			total,
+			passRate: total === 0 ? 0 : passed / total
+		},
+		evalFeedback: parsed.evalFeedback
+	};
+}
+//#endregion
+//#region src/grader/expectations.ts
+/**
+* Load expectations sidecar (YAML or JSON).
+*/
+/**
+* Load expectations sidecar (YAML or JSON).
+*
+* File format: `{ "<caseId>": ["expectation 1", ...], ... }`.
+*/
+async function loadExpectationsMap(path) {
+	const text = await readFile(path, "utf8");
+	const trimmed = path.trim().toLowerCase();
+	let raw;
+	if (trimmed.endsWith(".json")) raw = JSON.parse(text);
+	else raw = parse(text);
+	if (!raw || typeof raw !== "object") throw new Error(`expectations file must be an object mapping case ids to lists`);
+	const map = {};
+	for (const [caseId, value] of Object.entries(raw)) {
+		if (!Array.isArray(value)) throw new Error(`expectations for case "${caseId}" must be an array of strings`);
+		map[caseId] = value.map(String);
+	}
+	return map;
+}
+//#endregion
+//#region src/grader/transcript.ts
+/** Maximum characters per tool result embedded in grader transcripts. */
+const MAX_RESULT_CHARS = 4e3;
+/**
+* Render a {@link TrajectoryView} as markdown for LLM graders.
+*
+* Tool results are truncated at {@link MAX_RESULT_CHARS} to keep judge
+* prompts within reasonable token limits.
+*/
+function trajectoryToTranscript(view, prompt) {
+	const lines = [];
+	if (prompt) lines.push("## User prompt", "", prompt, "");
+	for (const turn of view.turns) {
+		lines.push(`## Assistant turn ${turn.turnIndex + 1}`, "");
+		if (turn.text) lines.push(turn.text, "");
+		for (const call of turn.toolCalls) {
+			lines.push(`[Tool call] ${call.name} (id=${call.callId})`);
+			lines.push(`Arguments: ${formatJson$1(call.args)}`);
+			if (call.result !== null) {
+				lines.push(`[Tool result] ${formatResult(call.result)}`);
+				if (call.isError) lines.push("(tool reported error)");
+			} else lines.push("[Tool result] (none observed)");
+			lines.push("");
+		}
+		if (turn.stopReason) lines.push(`Stop reason: ${turn.stopReason}`, "");
+	}
+	const finalInTurns = view.turns.some((t) => t.text === view.finalResponse);
+	if (view.finalResponse && !finalInTurns) lines.push("## Final response", "", view.finalResponse, "");
+	lines.push("## Session metadata", `session_id: ${view.meta.sessionId}`, `model: ${view.meta.model}`, `cwd: ${view.meta.cwd}`, `success: ${view.success}`, `tool_calls: ${view.toolCalls.length}`, `duration_ms: ${view.usage.durationMs}`, `input_tokens: ${view.usage.inputTokens}`, `output_tokens: ${view.usage.outputTokens}`);
+	return lines.join("\n").trimEnd();
+}
+/** Format unknown values as JSON for transcript embedding. */
+function formatJson$1(value) {
+	try {
+		return JSON.stringify(value);
+	} catch {
+		return String(value);
+	}
+}
+/** Format a tool result, truncating long string or JSON payloads. */
+function formatResult(result) {
+	if (typeof result === "string") return truncate(result);
+	return truncate(formatJson$1(result));
+}
+/** Truncate text with ellipsis when exceeding the transcript size budget. */
+function truncate(text) {
+	if (text.length <= MAX_RESULT_CHARS) return text;
+	return `${text.slice(0, MAX_RESULT_CHARS)}… (truncated)`;
+}
+//#endregion
+//#region src/eval-record/judge-metadata.ts
+/** Map harness grading adapter id to a stable judge identifier. */
+function judgeIdForAdapter(adapter) {
+	switch (adapter) {
+		case "codex": return "harness-eval/codex-grader";
+		case "claude-code": return "harness-eval/claude-grader";
+		default: return adapter ? `harness-eval/${adapter}-grader` : "harness-eval/claude-grader";
+	}
+}
+/** Build {@link JudgeInfo} from grading adapter and optional model override. */
+function resolveJudgeInfo(options) {
+	const adapter = options.adapter ?? "claude-code";
+	return {
+		id: options.id ?? judgeIdForAdapter(adapter),
+		model: options.model,
+		adapter
+	};
+}
+/** Derive judge metadata from a parsed grading YAML config. */
+function judgeInfoFromGradingConfig(config) {
+	return resolveJudgeInfo({
+		adapter: config.judge.adapter ?? "claude-code",
+		model: config.judge.model ?? config.judge.codex?.model ?? config.judge.claudeCode?.model
+	});
+}
+//#endregion
+//#region src/grader/grade-report.ts
+/**
+* Grade a harness-eval SuiteReport with outcome expectations (LLM judge).
+*/
+/**
+* Grade every repetition in a {@link SuiteReport} that has expectations.
+*
+* Expectations come from inline case fields or an optional sidecar YAML/JSON
+* map. Runs are concurrent under {@link GradeReportOptions.maxConcurrent}.
+*/
+async function gradeReport(report, options = {}) {
+	const expectationsMap = options.expectationsPath ? await loadExpectationsMap(options.expectationsPath) : {};
+	const gradeFn = options.gradeFn ?? (options.judgeAdapter === "codex" ? createCodexGrader({
+		binary: options.binary,
+		model: options.model,
+		timeoutMs: options.timeoutMs,
+		env: options.env,
+		cwd: options.cwd,
+		codex: options.codex
+	}) : createClaudeGrader({
+		binary: options.binary,
+		model: options.model,
+		timeoutMs: options.timeoutMs,
+		env: options.env,
+		cwd: options.cwd,
+		claudeCode: options.claudeCode
+	}));
+	const limit = createLimit(options.maxConcurrent ?? 2);
+	const tasks = [];
+	for (const cell of report.cells) {
+		const expectations = cell.expectations ?? expectationsMap[cell.caseId] ?? [];
+		if (expectations.length === 0) continue;
+		for (const rep of cell.repetitions) {
+			if (!rep.adapterResult) continue;
+			tasks.push({
+				cell,
+				rep,
+				expectations
+			});
+		}
+	}
+	const gradeStartTs = Date.now();
+	options.onProgress?.({
+		kind: "grade-start",
+		total: tasks.length
+	});
+	const results = await Promise.all(tasks.map(({ cell, rep, expectations }) => limit(async () => {
+		const start = Date.now();
+		const view = rep.adapterResult.view;
+		const prompt = cell.prompt ?? "";
+		const transcript = trajectoryToTranscript(view, prompt);
+		try {
+			const graded = await gradeFn({
+				prompt,
+				transcript,
+				expectations,
+				systemInstruction: options.systemInstruction
+			});
+			const result = {
+				caseId: cell.caseId,
+				cellLabel: cell.cell.label,
+				repetitionIndex: rep.repetitionIndex,
+				prompt,
+				expectations: graded.expectations,
+				summary: graded.summary,
+				evalFeedback: graded.evalFeedback,
+				graderError: graded.error,
+				durationMs: Date.now() - start
+			};
+			options.onProgress?.({
+				kind: "grade-complete",
+				caseId: result.caseId,
+				cellLabel: result.cellLabel,
+				repetitionIndex: result.repetitionIndex,
+				passed: result.summary.passed,
+				failed: result.summary.failed,
+				durationMs: result.durationMs,
+				graderError: result.graderError
+			});
+			return result;
+		} catch (err) {
+			const message = err instanceof Error ? err.message : String(err);
+			const result = {
+				caseId: cell.caseId,
+				cellLabel: cell.cell.label,
+				repetitionIndex: rep.repetitionIndex,
+				prompt,
+				expectations: expectations.map((text) => ({
+					text,
+					passed: false,
+					evidence: message
+				})),
+				summary: {
+					passed: 0,
+					failed: expectations.length,
+					total: expectations.length,
+					passRate: 0
+				},
+				graderError: message,
+				durationMs: Date.now() - start
+			};
+			options.onProgress?.({
+				kind: "grade-complete",
+				caseId: result.caseId,
+				cellLabel: result.cellLabel,
+				repetitionIndex: result.repetitionIndex,
+				passed: 0,
+				failed: expectations.length,
+				durationMs: result.durationMs,
+				graderError: message
+			});
+			return result;
+		}
+	})));
+	results.sort((a, b) => {
+		const keyA = `${a.caseId}::${a.cellLabel}::${a.repetitionIndex}`;
+		const keyB = `${b.caseId}::${b.cellLabel}::${b.repetitionIndex}`;
+		return keyA.localeCompare(keyB);
+	});
+	const totalExpectations = results.reduce((n, r) => n + r.summary.total, 0);
+	const passedExpectations = results.reduce((n, r) => n + r.summary.passed, 0);
+	options.onProgress?.({
+		kind: "grade-done",
+		durationMs: Date.now() - gradeStartTs,
+		totalExpectations,
+		passedExpectations
+	});
+	return {
+		gradedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		sourceReport: options.sourceReport ?? "",
+		gradingConfigPath: options.gradingConfigPath,
+		judge: resolveJudgeInfo({
+			adapter: options.judgeAdapter ?? "claude-code",
+			model: options.model
+		}),
+		results,
+		summary: {
+			passed: passedExpectations,
+			failed: totalExpectations - passedExpectations,
+			total: totalExpectations,
+			passRate: totalExpectations === 0 ? 0 : passedExpectations / totalExpectations
+		}
+	};
+}
+/** Load a suite report JSON file produced by `harness-eval run`. */
+async function loadSuiteReport(path) {
+	const text = await readFile(path, "utf8");
+	return JSON.parse(text);
+}
+//#endregion
+//#region src/grader/resolve-grade-options.ts
+/**
+* Merge standalone grading YAML with CLI flags (CLI wins).
+*/
+function resolveGradeOptions(fileConfig, cli = {}, configPath) {
+	const judge = fileConfig?.judge;
+	const adapter = judge?.adapter ?? "claude-code";
+	const claudeCode = judge?.claudeCode ?? {};
+	const codex = judge?.codex ?? {};
+	const adapterBlock = adapter === "codex" ? codex : claudeCode;
+	const binary = cli.binary ?? adapterBlock.binary;
+	const model = cli.model ?? judge?.model ?? adapterBlock.model;
+	if (adapter === "codex") return {
+		sourceReport: cli.sourceReport,
+		expectationsPath: cli.expectationsPath,
+		model,
+		binary,
+		timeoutMs: cli.timeoutMs ?? judge?.timeoutMs,
+		maxConcurrent: cli.maxConcurrent ?? judge?.maxConcurrent,
+		systemInstruction: judge?.system_instruction,
+		env: judge?.env,
+		cwd: judge?.cwd,
+		judgeAdapter: "codex",
+		codex: {
+			...codex,
+			binary: void 0,
+			model: void 0
+		},
+		gradingConfigPath: configPath
+	};
+	if (adapter !== "claude-code") throw new Error(`unsupported grading adapter "${adapter}" (supported: claude-code, codex)`);
+	return {
+		sourceReport: cli.sourceReport,
+		expectationsPath: cli.expectationsPath,
+		model,
+		binary,
+		timeoutMs: cli.timeoutMs ?? judge?.timeoutMs,
+		maxConcurrent: cli.maxConcurrent ?? judge?.maxConcurrent,
+		systemInstruction: judge?.system_instruction,
+		env: judge?.env,
+		cwd: judge?.cwd,
+		judgeAdapter: "claude-code",
+		claudeCode: {
+			...claudeCode,
+			binary: void 0,
+			model: void 0
+		},
+		gradingConfigPath: configPath
+	};
+}
+//#endregion
+//#region src/grader/format-console.ts
+const RESET$1 = "\x1B[0m";
+const GREEN$1 = "\x1B[32m";
+const RED$1 = "\x1B[31m";
+const DIM = "\x1B[2m";
+/**
+* Format a {@link SuiteGradingReport} for terminal output.
+*
+* @param color When true, emit ANSI status colors (default for TTY console).
+*/
+function formatGradingConsole(report, color = true) {
+	const lines = [];
+	if (report.results.length === 0) {
+		lines.push("No repetitions graded. Add expectations to the suite YAML or pass --expectations.");
+		return lines.join("\n");
+	}
+	for (const result of report.results) {
+		const status = result.summary.failed === 0 && !result.graderError ? color ? `${GREEN$1}PASS${RESET$1}` : "PASS" : color ? `${RED$1}FAIL${RESET$1}` : "FAIL";
+		lines.push(`${result.caseId} @ ${result.cellLabel} rep${result.repetitionIndex}  ${status}`);
+		if (result.graderError) lines.push(color ? `  ${RED$1}grader error: ${result.graderError}${RESET$1}` : `  grader error: ${result.graderError}`);
+		for (const exp of result.expectations) {
+			const marker = exp.passed ? color ? `${GREEN$1}✓${RESET$1}` : "✓" : color ? `${RED$1}✗${RESET$1}` : "✗";
+			lines.push(`  ├─ ${exp.text} ${marker}`);
+			if (!exp.passed || exp.evidence) lines.push(color ? `  │  ${DIM}${exp.evidence}${RESET$1}` : `  │  ${exp.evidence}`);
+		}
+		const pct = (result.summary.passRate * 100).toFixed(0);
+		lines.push(`  └─ ${result.summary.passed}/${result.summary.total} (${pct}%) expectations`);
+		lines.push("");
+	}
+	const overallPct = (report.summary.passRate * 100).toFixed(0);
+	lines.push(`Overall: ${report.summary.passed}/${report.summary.total} (${overallPct}%) expectations passed`);
+	return lines.join("\n").trimEnd();
+}
+/** True when every graded rep passed all expectations without grader errors. */
+function gradingReportPassed(report) {
+	return report.results.every((r) => !r.graderError && r.summary.failed === 0 && r.summary.total > 0);
+}
+//#endregion
+//#region src/eval-interchange/normalize.ts
+/**
+* Serialize tool arguments to the Vertex wire string format.
+*
+* Already-string inputs pass through unchanged (e.g. pre-serialized reference
+* steps). Objects and nullish values become JSON strings; empty input becomes `{}`.
+*
+* @param args - Tool arguments from harness or suite YAML.
+* @returns JSON string suitable for {@link ProtojsonToolCall.toolInput}.
+*/
+function serializeToolInput(args) {
+	if (typeof args === "string") return args;
+	return JSON.stringify(args ?? {});
+}
+/**
+* Normalize a tool name according to suite reference configuration.
+*
+* In `"bare"` mode, strips the MCP namespace prefix (`mcp__api__foo` → `foo`)
+* so reference trajectories authored with bare names match harness tool names.
+*
+* @param toolName - Raw tool name from harness or suite.
+* @param mode - `"harness"` preserves the name; `"bare"` strips after last `__`.
+*/
+function normalizeReferenceToolName(toolName, mode) {
+	if (mode !== "bare") return toolName;
+	const separator = toolName.lastIndexOf("__");
+	if (separator === -1) return toolName;
+	return toolName.slice(separator + 2);
+}
+/**
+* Convert a harness or suite trajectory into Vertex protojson wire format.
+*
+* `toolNameMode` controls MCP prefix stripping for every tool name in the
+* trajectory. Suite reference steps and predicted harness tool calls use the
+* same mode so comparisons stay consistent across metrics and instances.
+*
+* @param trajectory - Tool calls in harness or YAML reference shape.
+* @param options.toolNameMode - `"harness"` keeps full names; `"bare"` strips after last `__`.
+*/
+function toProtojsonTrajectory(trajectory, options = {}) {
+	const toolNameMode = options.toolNameMode ?? "harness";
+	return { toolCalls: trajectory.map((toolCall) => {
+		const name = "name" in toolCall ? toolCall.name : toolCall.tool_name;
+		const args = "args" in toolCall ? toolCall.args : toolCall.tool_input;
+		return {
+			toolName: normalizeReferenceToolName(name, toolNameMode),
+			toolInput: serializeToolInput(args)
+		};
+	}) };
+}
+//#endregion
+//#region src/eval-interchange/protojson/trajectory-instances.ts
+/**
+* Build Vertex Trajectory*Instance protojson wire objects.
+*
+* Each trajectory metric in Vertex EvaluateInstances expects a specific
+* protobuf message. This module constructs all six instance payloads from
+* one predicted/reference pair so callers can batch-upload via JSONL.
+*/
+/**
+* Build a pair instance with predicted and reference trajectories.
+*
+* Both sides use the same `referenceToolNameMode` so wire payloads align with
+* {@link toHarnessMetrics} and Vertex EvaluateInstances sees comparable names.
+* In `"bare"` mode, MCP prefixes are stripped on predicted and reference alike.
+*/
+function pairInstance(predicted, reference, referenceToolNameMode) {
+	return {
+		predictedTrajectory: toProtojsonTrajectory(predicted, { toolNameMode: referenceToolNameMode }),
+		referenceTrajectory: toProtojsonTrajectory(reference, { toolNameMode: referenceToolNameMode })
+	};
+}
+/**
+* Build all Trajectory*Instance payloads for one predicted/reference pair.
+*
+* Pair metrics (exact, in-order, any-order, precision, recall) share the
+* same trajectory pair; single-tool-use omits the reference trajectory
+* per Vertex API shape.
+*/
+function toTrajectoryInstances(options) {
+	const referenceToolNameMode = options.referenceToolNameMode ?? "harness";
+	const pair = pairInstance(options.predicted, options.reference, referenceToolNameMode);
+	return {
+		exactMatch: pair,
+		inOrderMatch: pair,
+		anyOrderMatch: pair,
+		precision: pair,
+		recall: pair,
+		singleToolUse: { predictedTrajectory: pair.predictedTrajectory }
+	};
+}
+/**
+* Convert suite reference steps to cell-level protojson trajectory export.
+*/
+function toReferenceTrajectory(reference, referenceToolNameMode = "harness") {
+	return toProtojsonTrajectory(reference, { toolNameMode: referenceToolNameMode });
+}
+/**
+* Map a trajectory instance key to the Vertex protobuf message type name.
+*
+* Used as `messageType` in {@link InstancesJsonlRow} for EvaluateInstances batching.
+*/
+function trajectoryInstanceMessageType(key) {
+	switch (key) {
+		case "exactMatch": return "TrajectoryExactMatchInstance";
+		case "inOrderMatch": return "TrajectoryInOrderMatchInstance";
+		case "anyOrderMatch": return "TrajectoryAnyOrderMatchInstance";
+		case "precision": return "TrajectoryPrecisionInstance";
+		case "recall": return "TrajectoryRecallInstance";
+		case "singleToolUse": return "TrajectorySingleToolUseInstance";
+	}
+}
+//#endregion
+//#region src/eval-interchange/protojson/evaluation-instance.ts
+/**
+* Build an EvaluationInstance protojson object from harness strings.
+*
+* Omitted fields are excluded from the output object rather than set to
+* empty wrappers — protojson omits unset optional fields.
+*
+* @param options.prompt - Case prompt sent to the agent.
+* @param options.response - Final agent response from the trajectory.
+* @param options.reference - Optional reference answer text (rare in harness eval).
+*/
+function toEvaluationInstance(options) {
+	const instance = {};
+	if (options.prompt !== void 0) instance.prompt = { text: options.prompt };
+	if (options.response !== void 0) instance.response = { text: options.response };
+	if (options.reference !== void 0) instance.reference = { text: options.reference };
+	return instance;
+}
+//#endregion
+//#region src/metrics/trajectory.ts
+/**
+* Trajectory-level metrics for comparing predicted and reference tool-call sequences.
+*
+* Aligns with Vertex AI EvaluationService trajectory metrics (exact match,
+* in-order, any-order, precision, recall, single tool use). Tool calls are
+* compared by `(tool_name, serialized tool_input)` identity after normalization.
+*
+* Binary metrics return 0 or 1; precision and recall return fractions in [0, 1].
+*/
+function normalizeToolCall(toolCall) {
+	if (typeof toolCall.tool_input === "string") return {
+		tool_name: toolCall.tool_name,
+		tool_input: toolCall.tool_input
+	};
+	return {
+		tool_name: toolCall.tool_name,
+		tool_input: serializeToolInput(toolCall.tool_input)
+	};
+}
+function normalizeTrajectory(trajectory) {
+	return trajectory.map(normalizeToolCall);
+}
+/** Stable composite key for multiset and equality checks. */
+function toolCallKey(toolCall) {
+	return `${toolCall.tool_name}\0${toolCall.tool_input}`;
+}
+/**
+* Count predicted tool calls that appear in reference (multiset intersection).
+*
+* Duplicate tool calls are matched one-for-one; order does not matter.
+*/
+function multisetIntersectionSize(predicted, reference) {
+	const refCounts = /* @__PURE__ */ new Map();
+	for (const toolCall of reference) {
+		const key = toolCallKey(toolCall);
+		refCounts.set(key, (refCounts.get(key) ?? 0) + 1);
+	}
+	let matched = 0;
+	for (const toolCall of predicted) {
+		const key = toolCallKey(toolCall);
+		const count = refCounts.get(key) ?? 0;
+		if (count > 0) {
+			matched += 1;
+			refCounts.set(key, count - 1);
+		}
+	}
+	return matched;
+}
+/**
+* Whether reference appears as a subsequence of predicted (order preserved).
+*
+* Extra predicted calls between reference steps are allowed (in-order match
+* semantics per Vertex).
+*/
+function isSubsequence(predicted, reference) {
+	let refIndex = 0;
+	for (const toolCall of predicted) {
+		if (refIndex >= reference.length) break;
+		if (toolCallKey(toolCall) === toolCallKey(reference[refIndex])) refIndex += 1;
+	}
+	return refIndex === reference.length;
+}
+function arraysEqual(left, right) {
+	if (left.length !== right.length) return false;
+	return left.every((toolCall, index) => {
+		const other = right[index];
+		return toolCallKey(toolCall) === toolCallKey(other);
+	});
+}
+/** Exact sequence equality after normalization. */
+function trajectoryExactMatch(predicted, reference) {
+	return arraysEqual(normalizeTrajectory(predicted), normalizeTrajectory(reference)) ? 1 : 0;
+}
+/** Reference is a subsequence of predicted (order preserved, extras allowed). */
+function trajectoryInOrderMatch(predicted, reference) {
+	return isSubsequence(normalizeTrajectory(predicted), normalizeTrajectory(reference)) ? 1 : 0;
+}
+/** Same multiset of tool calls; length must match. */
+function trajectoryAnyOrderMatch(predicted, reference) {
+	const predictedNorm = normalizeTrajectory(predicted);
+	const referenceNorm = normalizeTrajectory(reference);
+	if (predictedNorm.length !== referenceNorm.length) return 0;
+	const predictedKeys = predictedNorm.map(toolCallKey).sort();
+	const referenceKeys = referenceNorm.map(toolCallKey).sort();
+	return predictedKeys.every((key, index) => key === referenceKeys[index]) ? 1 : 0;
+}
+/**
+* Fraction of predicted tool calls that appear in reference (multiset).
+*
+* Returns 1 when both trajectories are empty.
+*/
+function trajectoryPrecision(predicted, reference) {
+	const predictedNorm = normalizeTrajectory(predicted);
+	if (predictedNorm.length === 0) return reference.length === 0 ? 1 : 0;
+	return multisetIntersectionSize(predictedNorm, normalizeTrajectory(reference)) / predictedNorm.length;
+}
+/**
+* Fraction of reference tool calls matched in predicted (multiset recall).
+*
+* Returns 1 when reference is empty and predicted is empty.
+*/
+function trajectoryRecall(predicted, reference) {
+	const referenceNorm = normalizeTrajectory(reference);
+	if (referenceNorm.length === 0) return predicted.length === 0 ? 1 : 0;
+	return multisetIntersectionSize(normalizeTrajectory(predicted), referenceNorm) / referenceNorm.length;
+}
+/** Both trajectories have exactly one call and they match. */
+function trajectorySingleToolUse(predicted, reference) {
+	const predictedNorm = normalizeTrajectory(predicted);
+	const referenceNorm = normalizeTrajectory(reference);
+	if (predictedNorm.length !== 1 || referenceNorm.length !== 1) return 0;
+	return toolCallKey(predictedNorm[0]) === toolCallKey(referenceNorm[0]) ? 1 : 0;
+}
+/** Compute all trajectory metrics in one pass. */
+function computeTrajectoryMetrics(predicted, reference) {
+	return {
+		trajectory_exact_match: trajectoryExactMatch(predicted, reference),
+		trajectory_in_order_match: trajectoryInOrderMatch(predicted, reference),
+		trajectory_any_order_match: trajectoryAnyOrderMatch(predicted, reference),
+		trajectory_precision: trajectoryPrecision(predicted, reference),
+		trajectory_recall: trajectoryRecall(predicted, reference),
+		trajectory_single_tool_use: trajectorySingleToolUse(predicted, reference)
+	};
+}
+/**
+* Parse a wire tool_input string to JSON, or return the raw string on failure.
+*
+* Exported for tool-call metrics that need structured arg comparison.
+*/
+function parseToolInput(toolInput) {
+	try {
+		return JSON.parse(toolInput);
+	} catch {
+		return toolInput;
+	}
+}
+//#endregion
+//#region src/eval-interchange/protojson/harness-metrics.ts
+/**
+* Harness-owned trajectory metric scores in Vertex camelCase field names.
+*
+* Wraps {@link computeTrajectoryMetrics} for envelope export. External
+* systems can compare harness-precomputed scores against Vertex EvaluateInstances
+* results without reimplementing trajectory matching logic.
+*/
+/**
+* Compute trajectory metrics and map snake_case keys to Vertex camelCase.
+*
+* When `referenceToolNameMode` is `"bare"`, both predicted and reference tool
+* names are stripped to the suffix after the last `__` so suite reference steps
+* authored with bare names (e.g. `ListLandingZones`) match harness MCP names
+* (e.g. `mcp__plugin__ListLandingZones`).
+*
+* @param predicted - Tool calls from the harness trajectory view.
+* @param reference - Reference steps from suite YAML.
+* @param options.referenceToolNameMode - Name normalization mode from suite YAML.
+*/
+function toHarnessMetrics(predicted, reference, options = {}) {
+	const referenceToolNameMode = options.referenceToolNameMode ?? "harness";
+	const metrics = computeTrajectoryMetrics(predicted.map((toolCall) => ({
+		tool_name: normalizeReferenceToolName(toolCall.name, referenceToolNameMode),
+		tool_input: toolCall.args
+	})), reference.map((step) => ({
+		tool_name: normalizeReferenceToolName(step.tool_name, referenceToolNameMode),
+		tool_input: step.tool_input
+	})));
+	return {
+		trajectoryExactMatch: metrics.trajectory_exact_match,
+		trajectoryInOrderMatch: metrics.trajectory_in_order_match,
+		trajectoryAnyOrderMatch: metrics.trajectory_any_order_match,
+		trajectoryPrecision: metrics.trajectory_precision,
+		trajectoryRecall: metrics.trajectory_recall,
+		trajectorySingleToolUse: metrics.trajectory_single_tool_use
+	};
+}
+//#endregion
+//#region src/eval-interchange/enrich.ts
+/**
+* Enrich eval repetitions with Vertex protojson interchange fields.
+*
+* Called during envelope build for each successful repetition. Adds
+* `evaluationInstance`, optional `trajectoryInstances` / `harnessMetrics`
+* when a suite reference exists, and Vertex-style `latencySeconds` / `failure`
+* flags derived from trajectory success.
+*/
+/** Extract reference steps from suite config when present. */
+function referenceSteps(reference) {
+	return reference?.steps;
+}
+/**
+* Attach Vertex protojson interchange fields to one {@link EvalRepetition}.
+*
+* When no trajectory exists (adapter error), sets `failure: 1` and skips
+* protojson payloads. Trajectory instances and harness metrics are only
+* computed when the suite defines a non-empty reference trajectory.
+*
+* @param repetition - Base repetition from the runner (trajectory, assertions, grades).
+* @param options.prompt - Case prompt for EvaluationInstance.
+* @param options.reference - Suite reference trajectory config, if any.
+*/
+function enrichRepetitionWithProtojson(repetition, options = {}) {
+	if (!repetition.trajectory) return {
+		...repetition,
+		failure: 1
+	};
+	const predicted = repetition.trajectory.toolCalls;
+	const referenceStepsList = referenceSteps(options.reference);
+	const referenceToolNameMode = options.reference?.tool_name_mode ?? "harness";
+	const enriched = {
+		...repetition,
+		evaluationInstance: toEvaluationInstance({
+			prompt: options.prompt,
+			response: repetition.trajectory.finalResponse
+		}),
+		latencySeconds: repetition.trajectory.usage.durationMs / 1e3,
+		failure: repetition.trajectory.success ? 0 : 1
+	};
+	if (referenceStepsList?.length) {
+		enriched.trajectoryInstances = toTrajectoryInstances({
+			predicted,
+			reference: referenceStepsList,
+			referenceToolNameMode
+		});
+		enriched.harnessMetrics = toHarnessMetrics(predicted, referenceStepsList, { referenceToolNameMode });
+	}
+	return enriched;
+}
+//#endregion
+//#region src/eval-record/build.ts
+/**
+* Build {@link EvalRunEnvelope} from harness-eval run and grading reports.
+*
+* This is the canonical export path from in-process or on-disk {@link SuiteReport}
+* JSON into the cross-harness eval record contract. It stitches together:
+*
+*   - Behavioral assertion results from the runner
+*   - Optional outcome grades from the LLM grader
+*   - Vertex protojson interchange fields via {@link enrichRepetitionWithProtojson}
+*   - Optional artifacts (transcript, raw stream-json) controlled by build options
+*
+* Downstream consumers include CI gates, databases, and the `harness-eval envelope`
+* CLI projection commands.
+*/
+/**
+* Pull raw stream-json events from an adapter result when the adapter exposes them.
+*
+* Adapters may attach `rawEvents` for debug-only envelope export; this helper
+* avoids coupling the builder to a specific adapter result type.
+*/
+function extractRawEvents(adapterResult) {
+	if (adapterResult !== null && typeof adapterResult === "object" && "rawEvents" in adapterResult && Array.isArray(adapterResult.rawEvents)) return adapterResult.rawEvents;
+}
+/**
+* Derive cell-level outcome pass from graded repetitions.
+*
+* Returns `undefined` when no repetition was graded (outcome gate not applicable).
+* When graded, every repetition must have zero failed expectations and no grader error.
+*
+* @param _caseId - Reserved for future per-case outcome rules; unused today.
+* @param _cellLabel - Reserved for future per-cell outcome rules; unused today.
+*/
+function outcomePassForCell(_caseId, _cellLabel, repetitions) {
+	const graded = repetitions.filter((r) => r.outcomeGrades);
+	if (graded.length === 0) return void 0;
+	return graded.every((r) => r.outcomeGrades.error === void 0 && r.outcomeGrades.summary.failed === 0);
+}
+/** Resolve judge metadata for envelope export (explicit options win). */
+async function resolveEnvelopeJudge(options) {
+	if (options.grading?.judge) return options.grading.judge;
+	if (options.gradingConfigPath) try {
+		return judgeInfoFromGradingConfig(await loadGradingConfig(resolve(options.gradingConfigPath)));
+	} catch {}
+	return resolveJudgeInfo({ adapter: "claude-code" });
+}
+/** Path to pass to {@link loadSuite} (directory layout uses the suite folder). */
+async function resolveSuiteLoadPath(suitePath) {
+	const abs = resolve(suitePath);
+	if (basename(abs) === "suite.yaml") return dirname(abs);
+	try {
+		if ((await stat(abs)).isDirectory()) return abs;
+	} catch {}
+	return abs;
+}
+/** Read suite YAML bytes for content hashing. */
+async function readSuiteYamlContent(suitePath) {
+	const loadPath = await resolveSuiteLoadPath(suitePath);
+	return readFile(basename(resolve(suitePath)) === "suite.yaml" ? resolve(suitePath) : join(loadPath, "suite.yaml"), "utf8");
+}
+async function resolveEnvelopeHarnessAdapter(options) {
+	if (options.harnessAdapter) return options.harnessAdapter;
+	if (options.suitePath) try {
+		const suite = await loadSuite(await resolveSuiteLoadPath(options.suitePath));
+		if (suite.adapter) return suite.adapter;
+	} catch {}
+	return "claude-code";
+}
+/**
+* Convert a {@link SuiteReport} (and optional grading) into a versioned
+* {@link EvalRunEnvelope} for storage or API handoff.
+*
+* @param report - Runner output for one suite execution.
+* @param options - Provenance, grading merge, and artifact inclusion flags.
+* @returns A fully populated envelope with protojson interchange fields on each repetition.
+*/
+function buildEvalRunEnvelope(report, options = {}) {
+	const includeTranscript = options.includeTranscript !== false;
+	const includeRaw = options.includeRawStreamEvents === true;
+	const judge = options.grading?.judge ?? resolveJudgeInfo({ adapter: "claude-code" });
+	const cells = report.cells.map((cell) => {
+		const prompt = cell.prompt ?? "";
+		const referenceTrajectoryConfig = cell.reference_trajectory;
+		const referenceTrajectory = referenceTrajectoryConfig ? toReferenceTrajectory(referenceTrajectoryConfig.steps, referenceTrajectoryConfig.tool_name_mode ?? "harness") : void 0;
+		const repetitions = cell.repetitions.map((rep) => {
+			const base = {
+				repetitionIndex: rep.repetitionIndex,
+				durationMs: rep.durationMs,
+				assertionResults: rep.assertionResults
+			};
+			if (rep.error) {
+				base.error = {
+					message: rep.error.message,
+					diagnostics: rep.error.diagnostics
+				};
+				return base;
+			}
+			if (rep.adapterResult) {
+				base.trajectory = {
+					...rep.adapterResult.view,
+					schemaVersion: "1.0"
+				};
+				base.diagnostics = rep.adapterResult.diagnostics;
+				const artifacts = {};
+				if (includeTranscript) artifacts.transcript = trajectoryToTranscript(rep.adapterResult.view, prompt);
+				if (includeRaw) {
+					const raw = extractRawEvents(rep.adapterResult);
+					if (raw) artifacts.rawStreamEvents = raw;
+				}
+				if (Object.keys(artifacts).length > 0) base.artifacts = artifacts;
+			}
+			const graded = options.grading?.results.find((r) => r.caseId === cell.caseId && r.cellLabel === cell.cell.label && r.repetitionIndex === rep.repetitionIndex);
+			if (graded) base.outcomeGrades = {
+				judge,
+				expectations: graded.expectations,
+				summary: graded.summary,
+				evalFeedback: graded.evalFeedback,
+				error: graded.graderError
+			};
+			return enrichRepetitionWithProtojson(base, {
+				prompt,
+				reference: referenceTrajectoryConfig
+			});
+		});
+		return {
+			caseId: cell.caseId,
+			category: cell.category,
+			notes: cell.notes,
+			prompt: cell.prompt,
+			expectations: cell.expectations,
+			referenceTrajectory,
+			humanRatings: cell.human_ratings,
+			cellLabel: cell.cell.label,
+			axes: cell.cell.axes,
+			assertionStats: cell.assertionStats,
+			adapterErrors: cell.adapterErrors,
+			behavioralPass: cell.passed,
+			outcomePass: outcomePassForCell(cell.caseId, cell.cell.label, repetitions),
+			repetitions
+		};
+	});
+	const cellsPassed = cells.filter((c) => c.behavioralPass).length;
+	const gradedCells = cells.filter((c) => c.outcomePass !== void 0);
+	const outcomePass = gradedCells.length > 0 ? gradedCells.every((c) => c.outcomePass === true) : void 0;
+	return {
+		schemaVersion: "1.0",
+		runId: options.runId ?? randomUUID(),
+		startedAt: report.startedAt,
+		durationMs: report.durationMs,
+		suite: options.suite,
+		harness: {
+			adapter: options.harness?.adapter ?? "claude-code",
+			frameworkVersion: options.harness?.frameworkVersion,
+			harnessVersion: options.harness?.harnessVersion
+		},
+		provenance: options.provenance,
+		summary: {
+			cellsTotal: cells.length,
+			cellsPassed,
+			behavioralPass: cellsPassed === cells.length,
+			outcomePass
+		},
+		cells
+	};
+}
+/**
+* Build an envelope from on-disk runner and grader JSON artifacts.
+*
+* Reads `reportPath` as a {@link SuiteReport}. When `gradingPath` is set, merges
+* outcome grades from a {@link SuiteGradingReport}. When `suitePath` is set,
+* attaches suite URI and SHA-256 content hash for reproducibility.
+*
+* @param reportPath - Path to the suite run report JSON from `harness-eval run`.
+* @param options - Same build options as {@link buildEvalRunEnvelope}, plus file paths.
+*/
+async function buildEvalRunEnvelopeFromFiles(reportPath, options = {}) {
+	const reportText = await readFile(reportPath, "utf8");
+	const report = JSON.parse(reportText);
+	const harnessAdapter = await resolveEnvelopeHarnessAdapter({
+		harnessAdapter: options.harness?.adapter,
+		suitePath: options.suitePath
+	});
+	let grading = options.grading;
+	if (options.gradingPath) {
+		const gradingText = await readFile(options.gradingPath, "utf8");
+		const parsed = JSON.parse(gradingText);
+		const judge = parsed.judge ?? await resolveEnvelopeJudge({ gradingConfigPath: parsed.gradingConfigPath });
+		grading = {
+			gradedAt: parsed.gradedAt,
+			sourceReport: parsed.sourceReport,
+			results: parsed.results,
+			judge
+		};
+	}
+	let suite = options.suite;
+	if (options.suitePath) {
+		const content = await readSuiteYamlContent(options.suitePath);
+		suite = {
+			...suite,
+			uri: options.suitePath,
+			contentHash: createHash("sha256").update(content).digest("hex")
+		};
+	}
+	return buildEvalRunEnvelope(report, {
+		...options,
+		suite,
+		grading,
+		harness: {
+			...options.harness,
+			adapter: harnessAdapter
+		}
+	});
+}
+//#endregion
+//#region src/eval-interchange/projections.ts
+/** Trajectory instance keys emitted in stable order for JSONL export. */
+const TRAJECTORY_INSTANCE_KEYS = [
+	"exactMatch",
+	"inOrderMatch",
+	"anyOrderMatch",
+	"precision",
+	"recall",
+	"singleToolUse"
+];
+/**
+* Flatten one repetition into a trajectory dataset row.
+*
+* Pulls prompt from the cell, response from evaluationInstance, and falls
+* back to duration-based latency when enrich did not set latencySeconds.
+*/
+function repetitionToDatasetRow(cell, repetition) {
+	return {
+		caseId: cell.caseId,
+		repetitionIndex: repetition.repetitionIndex,
+		prompt: cell.prompt,
+		response: repetition.evaluationInstance?.response?.text,
+		evaluationInstance: repetition.evaluationInstance,
+		latencySeconds: repetition.latencySeconds ?? repetition.durationMs / 1e3,
+		failure: repetition.failure ?? (repetition.trajectory?.success ? 0 : 1),
+		humanRatings: cell.humanRatings
+	};
+}
+/**
+* Expand one repetition into type-tagged instance rows for EvaluateInstances.
+*
+* Returns an empty array when the repetition has no reference trajectory
+* (and therefore no trajectoryInstances block).
+*/
+function repetitionToInstanceRows(cell, repetition) {
+	if (!repetition.trajectoryInstances) return [];
+	const rows = [];
+	for (const key of TRAJECTORY_INSTANCE_KEYS) {
+		const instance = repetition.trajectoryInstances[key];
+		if (!instance) continue;
+		rows.push({
+			messageType: trajectoryInstanceMessageType(key),
+			caseId: cell.caseId,
+			repetitionIndex: repetition.repetitionIndex,
+			instance
+		});
+	}
+	return rows;
+}
+/**
+* Trajectory projection — all repetitions in the envelope as dataset rows.
+*/
+function toTrajectory(envelope) {
+	const rows = [];
+	for (const cell of envelope.cells) for (const repetition of cell.repetitions) rows.push(repetitionToDatasetRow(cell, repetition));
+	return rows;
+}
+/**
+* Instances projection — all trajectory metric instances as JSONL rows.
+*/
+function toInstancesJsonl(envelope) {
+	const rows = [];
+	for (const cell of envelope.cells) for (const repetition of cell.repetitions) rows.push(...repetitionToInstanceRows(cell, repetition));
+	return rows;
+}
+//#endregion
+//#region src/pipeline/resolve-inputs.ts
+/**
+* Resolve pipeline step inputs and outputs with precedence rules.
+*
+* Precedence: CLI override > explicit YAML > prior step in this run > default path on disk > error.
+*/
+/** Resolve absolute paths for enabled pipeline steps. */
+async function resolvePipelineInputs(options) {
+	const { suitePath, suiteDir, pipeline, steps, overrides } = options;
+	const executed = options.executed ?? {};
+	const stepSet = new Set(steps);
+	const resolved = { suitePath: resolve(suitePath) };
+	const defaultRunOutput = resolve(suiteDir, pipeline.run?.output ?? DEFAULT_PIPELINE_OUTPUTS.run);
+	const defaultGradeOutput = resolve(suiteDir, pipeline.grade?.output ?? DEFAULT_PIPELINE_OUTPUTS.grade);
+	if (stepSet.has("run") && pipeline.run) resolved.run = {
+		output: resolve(suiteDir, overrides?.run?.output ?? pipeline.run.output),
+		maxConcurrent: overrides?.run?.maxConcurrent ?? pipeline.run.maxConcurrent
+	};
+	if (stepSet.has("grade") && pipeline.grade) resolved.grade = {
+		input: await resolveReportPath({
+			explicit: overrides?.grade?.input ?? pipeline.grade.input,
+			executedOutput: executed.run?.output,
+			defaultPath: defaultRunOutput,
+			label: "grade input (report)"
+		}),
+		output: resolve(suiteDir, overrides?.grade?.output ?? pipeline.grade.output),
+		maxConcurrent: overrides?.grade?.maxConcurrent ?? pipeline.grade.maxConcurrent
+	};
+	if (stepSet.has("envelope") && pipeline.envelope) resolved.envelope = {
+		report: await resolveReportPath({
+			explicit: overrides?.envelope?.report ?? pipeline.envelope.report,
+			executedOutput: executed.run?.output,
+			defaultPath: defaultRunOutput,
+			label: "envelope report"
+		}),
+		grading: await resolveOptionalGradingPath({
+			explicit: overrides?.envelope?.grading ?? pipeline.envelope.grading,
+			executedOutput: executed.grade?.output,
+			defaultPath: defaultGradeOutput
+		}),
+		output: resolve(suiteDir, overrides?.envelope?.output ?? pipeline.envelope.output),
+		projection: overrides?.envelope?.projection ?? pipeline.envelope.projection ?? "envelope",
+		includeRawStreamEvents: pipeline.envelope.includeRawStreamEvents ?? false,
+		noTranscript: pipeline.envelope.noTranscript ?? false
+	};
+	return resolved;
+}
+/**
+* Resolve a required report path: explicit override → prior step output → default on disk.
+* Throws when none of the above exist.
+*/
+async function resolveReportPath(options) {
+	if (options.explicit) return resolve(options.explicit);
+	if (options.executedOutput) return resolve(options.executedOutput);
+	if (await pathExists(options.defaultPath)) return options.defaultPath;
+	throw new ConfigError(`pipeline: could not resolve ${options.label}; specify an explicit path or run the run step first`, options.defaultPath);
+}
+/** Resolve optional grading path; returns undefined when grading was not run and file is absent. */
+async function resolveOptionalGradingPath(options) {
+	if (options.explicit) return resolve(options.explicit);
+	if (options.executedOutput) return resolve(options.executedOutput);
+	if (await pathExists(options.defaultPath)) return options.defaultPath;
+}
+async function pathExists(filePath) {
+	try {
+		await stat(filePath);
+		return true;
+	} catch {
+		return false;
+	}
+}
+/**
+* Resolve a grading artifact path from a unified suite's `pipeline:` block.
+*
+* Used by `harness-eval envelope --suite` when `--grading` is omitted (spec C-7).
+* Checks `pipeline.envelope.grading` then default `pipeline.grade.output` on disk.
+*/
+async function resolveGradingArtifactFromSuite(suitePath) {
+	let doc;
+	try {
+		doc = await loadSuiteDocument(suitePath);
+	} catch {
+		return;
+	}
+	if (!doc.pipeline) return void 0;
+	const explicit = doc.pipeline.envelope?.grading;
+	if (explicit && await pathExists(explicit)) return explicit;
+	const defaultGrade = doc.pipeline.grade?.output;
+	if (defaultGrade && await pathExists(defaultGrade)) return defaultGrade;
+}
+/** Parse `--steps run,grade,envelope` against configured pipeline keys. */
+function parsePipelineSteps(pipeline, stepsArg) {
+	const configured = [];
+	if (pipeline.run !== void 0) configured.push("run");
+	if (pipeline.grade !== void 0) configured.push("grade");
+	if (pipeline.envelope !== void 0) configured.push("envelope");
+	if (configured.length === 0) throw new ConfigError("pipeline block has no steps configured");
+	if (!stepsArg) return configured;
+	const validStepNames = /* @__PURE__ */ new Set([
+		"run",
+		"grade",
+		"envelope"
+	]);
+	const requested = stepsArg.split(",").map((s) => s.trim()).filter(Boolean);
+	for (const step of requested) {
+		if (!validStepNames.has(step)) throw new ConfigError(`unknown pipeline step "${step}"; valid steps are: run, grade, envelope`);
+		if (!configured.includes(step)) throw new ConfigError(`pipeline step "${step}" is not configured in suite.yaml`);
+	}
+	const requestedSet = new Set(requested);
+	return configured.filter((step) => requestedSet.has(step));
+}
+/** Parent directory of suite.yaml. */
+function suiteDirectoryFromPath(suitePath) {
+	return dirname(resolve(suitePath));
+}
+//#endregion
+//#region src/cli/args.ts
+/** Parse process argv into command, positional args, and options. */
+function parseArgs(argv) {
+	const positional = [];
+	const options = {};
+	let command;
+	const args = [...argv];
+	if (args.length > 0 && !args[0].startsWith("-")) command = args.shift();
+	for (let i = 0; i < args.length; i++) {
+		const arg = args[i];
+		if (arg === "--") {
+			positional.push(...args.slice(i + 1));
+			break;
+		}
+		if (arg.startsWith("--")) {
+			const key = arg.slice(2);
+			const next = args[i + 1];
+			if (next && !next.startsWith("-")) {
+				options[key] = next;
+				i++;
+			} else options[key] = true;
+		} else if (arg.startsWith("-") && arg.length === 2) {
+			const key = arg.slice(1);
+			const next = args[i + 1];
+			if (next && !next.startsWith("-")) {
+				options[key] = next;
+				i++;
+			} else options[key] = true;
+		} else positional.push(arg);
+	}
+	return {
+		command,
+		positional,
+		options
+	};
+}
+/** Return a string option value, or undefined when absent or boolean. */
+function getOption(options, name) {
+	const v = options[name];
+	return typeof v === "string" ? v : void 0;
+}
+/** Parse an integer option with fallback when absent or non-numeric. */
+function getOptionInt(options, name, defaultValue) {
+	const v = getOption(options, name);
+	if (v === void 0) return defaultValue;
+	const n = Number.parseInt(v, 10);
+	if (!Number.isFinite(n)) return defaultValue;
+	return n;
+}
+/** True when a boolean flag is set or explicitly `"true"`. */
+function hasOption(options, name) {
+	const v = options[name];
+	return v === true || typeof v === "string" && v === "true";
+}
+//#endregion
+//#region src/cli/commands/envelope.ts
+/**
+* `harness-eval envelope` — build EvalRunEnvelope and interchange projections.
+*
+* Reads a suite run report (and optional grading JSON), builds a versioned
+* {@link EvalRunEnvelope}, and serializes one of three projections:
+*
+*   - `envelope` — full nested JSON document (default)
+*   - `trajectory` — JSONL of {@link EvalDatasetRow} per repetition
+*   - `instances` — JSONL of {@link InstancesJsonlRow} for Vertex batch upload
+*
+* Exit code 0 when behavioral pass, 1 when any cell failed assertions.
+*/
+const PROJECTIONS = /* @__PURE__ */ new Set([
+	"envelope",
+	"trajectory",
+	"instances"
+]);
+/**
+* Parse and validate `--projection` CLI flag.
+*
+* @returns `"envelope"` when omitted; `undefined` when value is invalid.
+*/
+function parseEnvelopeProjection(value) {
+	if (value === void 0) return "envelope";
+	if (PROJECTIONS.has(value)) return value;
+}
+/**
+* Serialize an envelope to stdout/file string for the chosen projection.
+*
+* Trajectory and instances projections emit NDJSON (one JSON object per line).
+*/
+function serializeEnvelopeProjection(envelope, projection) {
+	switch (projection) {
+		case "trajectory": return `${toTrajectory(envelope).map((row) => JSON.stringify(row)).join("\n")}\n`;
+		case "instances": return `${toInstancesJsonl(envelope).map((row) => JSON.stringify(row)).join("\n")}\n`;
+		default: return `${JSON.stringify(envelope, null, 2)}\n`;
+	}
+}
+/** Read harness-eval package version for envelope harness.frameworkVersion. */
+async function readFrameworkVersion() {
+	try {
+		const text = await readFile(join(dirname(fileURLToPath(import.meta.url)), "../../../package.json"), "utf8");
+		return JSON.parse(text).version;
+	} catch {
+		return;
+	}
+}
+/**
+* CLI entry point for the `envelope` subcommand.
+*
+* @returns Process exit code: 0 on behavioral pass, 1 on failure, 2 on usage/error.
+*/
+async function envelopeCommand(args) {
+	const reportPath = args.positional[0];
+	if (!reportPath) {
+		console.error("usage: harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]");
+		return 2;
+	}
+	const outputPath = getOption(args.options, "output");
+	const suitePath = getOption(args.options, "suite");
+	let gradingPath = getOption(args.options, "grading");
+	if (!gradingPath && suitePath) gradingPath = await resolveGradingArtifactFromSuite(suitePath);
+	const projection = parseEnvelopeProjection(getOption(args.options, "projection"));
+	if (!projection) {
+		console.error("invalid --projection; expected envelope, trajectory, or instances");
+		return 2;
+	}
+	let envelope;
+	try {
+		const frameworkVersion = await readFrameworkVersion();
+		envelope = await buildEvalRunEnvelopeFromFiles(reportPath, {
+			gradingPath,
+			suitePath,
+			includeTranscript: !hasOption(args.options, "no-transcript"),
+			includeRawStreamEvents: hasOption(args.options, "include-raw-stream-events"),
+			harness: { frameworkVersion }
+		});
+	} catch (err) {
+		console.error(err instanceof Error ? err.message : String(err));
+		return 2;
+	}
+	const serialized = serializeEnvelopeProjection(envelope, projection);
+	if (outputPath) await writeFile(outputPath, serialized, "utf8");
+	else process.stdout.write(serialized);
+	return envelope.summary.behavioralPass ? 0 : 1;
+}
+//#endregion
+//#region src/pipeline/run-pipeline.ts
+/**
+* Orchestrate run → grade → envelope pipeline steps.
+*/
+/** Execute configured pipeline steps in order; stop on first failure. */
+async function runPipeline(doc, options = {}) {
+	if (!doc.pipeline) throw new ConfigError("suite document has no pipeline block", doc.suitePath);
+	const steps = parsePipelineSteps(doc.pipeline, options.steps);
+	const suiteDir = suiteDirectoryFromPath(doc.suitePath);
+	const executed = {};
+	let runReport;
+	let exitCode = 0;
+	for (const step of steps) {
+		const resolved = await resolvePipelineInputs({
+			suitePath: doc.suitePath,
+			suiteDir,
+			pipeline: doc.pipeline,
+			steps: [step],
+			executed,
+			overrides: options.overrides
+		});
+		if (step === "run" && resolved.run) {
+			const adapter = getAdapter(doc.suite.adapter ?? "claude-code");
+			runReport = await runSuite(doc.suite, {
+				adapter,
+				maxConcurrent: resolved.run.maxConcurrent ?? options.maxConcurrent ?? 4,
+				onProgress: options.onRunProgress
+			});
+			await writeFile(resolved.run.output, JSON.stringify(runReport, null, 2), "utf8");
+			executed.run = { output: resolved.run.output };
+			if (!runReport.cells.every((cell) => cell.passed)) return {
+				exitCode: 1,
+				stepsRun: steps.slice(0, steps.indexOf(step) + 1),
+				runReport
+			};
+			continue;
+		}
+		if (step === "grade" && resolved.grade) {
+			if (!doc.judge) throw new ConfigError("grade step requires inline judge: block in suite.yaml", doc.suitePath);
+			const gradeOptions = resolveGradeOptions({ judge: doc.judge }, {
+				sourceReport: resolved.grade.input,
+				maxConcurrent: resolved.grade.maxConcurrent
+			}, doc.suitePath);
+			const grading = await gradeReport(await loadSuiteReport(resolved.grade.input), {
+				...gradeOptions,
+				onProgress: options.onGradeProgress
+			});
+			await writeFile(resolved.grade.output, JSON.stringify(grading, null, 2), "utf8");
+			executed.grade = {
+				input: resolved.grade.input,
+				output: resolved.grade.output
+			};
+			if (!gradingReportPassed(grading)) return {
+				exitCode: 1,
+				stepsRun: steps.slice(0, steps.indexOf(step) + 1),
+				runReport
+			};
+			continue;
+		}
+		if (step === "envelope" && resolved.envelope) {
+			const envelope = await buildEvalRunEnvelopeFromFiles(resolved.envelope.report, {
+				gradingPath: resolved.envelope.grading,
+				suitePath: doc.suitePath,
+				includeTranscript: !resolved.envelope.noTranscript,
+				includeRawStreamEvents: resolved.envelope.includeRawStreamEvents,
+				harness: { frameworkVersion: options.frameworkVersion }
+			});
+			const serialized = serializeEnvelopeProjection(envelope, resolved.envelope.projection);
+			await writeFile(resolved.envelope.output, serialized, "utf8");
+			const behavioralFail = !envelope.summary.behavioralPass;
+			const outcomeFail = envelope.summary.outcomePass !== void 0 && !envelope.summary.outcomePass;
+			if (behavioralFail || outcomeFail) return {
+				exitCode: 1,
+				stepsRun: steps.slice(0, steps.indexOf(step) + 1),
+				runReport
+			};
+			continue;
+		}
+	}
+	return {
+		exitCode,
+		stepsRun: steps,
+		runReport
+	};
+}
+//#endregion
+//#region src/otel/attributes.ts
+/** Build a string-typed OTLP attribute. */
+function strAttr(key, value) {
+	return {
+		key,
+		value: { stringValue: value }
+	};
+}
+/** Build an integer-typed OTLP attribute (stored as decimal string). */
+function intAttr(key, value) {
+	return {
+		key,
+		value: { intValue: String(value) }
+	};
+}
+/** Build a boolean-typed OTLP attribute. */
+function boolAttr(key, value) {
+	return {
+		key,
+		value: { boolValue: value }
+	};
+}
+/** Build a JSON-serialized string attribute (common for message arrays). */
+function jsonAttr(key, value) {
+	return {
+		key,
+		value: { stringValue: JSON.stringify(value) }
+	};
+}
+//#endregion
+//#region src/otel/messages.ts
+/**
+* Map harness stop reasons to GenAI semconv finish_reason values.
+*
+* Unknown reasons pass through unchanged for forward compatibility.
+*/
+function mapStopReason(reason) {
+	if (!reason) return void 0;
+	switch (reason) {
+		case "end_turn": return "stop";
+		case "tool_use": return "tool_calls";
+		case "max_tokens": return "length";
+		case "stop_sequence": return "stop";
+		default: return reason;
+	}
+}
+/** Build a tool_call part from a {@link ToolCall}. */
+function toolCallPart(call) {
+	return {
+		type: "tool_call",
+		id: call.callId,
+		name: call.name,
+		arguments: call.args ?? {}
+	};
+}
+/** Build a tool_call_response part from a {@link ToolCall} result. */
+function toolResponsePart(call) {
+	return {
+		type: "tool_call_response",
+		id: call.callId,
+		result: call.result
+	};
+}
+/** Convert one assistant turn to a GenAI semconv assistant message. */
+function assistantMessageFromTurn(turn) {
+	const parts = [];
+	if (turn.text) parts.push({
+		type: "text",
+		content: turn.text
+	});
+	for (const call of turn.toolCalls) parts.push(toolCallPart(call));
+	const finish = mapStopReason(turn.stopReason);
+	return {
+		role: "assistant",
+		parts,
+		...finish ? { finish_reason: finish } : {}
+	};
+}
+/** Aggregate tool results from a turn into a single tool-role message, if any. */
+function toolResultsMessage(calls) {
+	const parts = calls.filter((c) => c.result !== null).map((c) => toolResponsePart(c));
+	if (parts.length === 0) return null;
+	return {
+		role: "tool",
+		parts
+	};
+}
+/**
+* Input history before the assistant turn at `turnIndex`.
+*/
+function inputMessagesBeforeTurn(view, turnIndex, prompt) {
+	const messages = [];
+	if (prompt) messages.push({
+		role: "user",
+		parts: [{
+			type: "text",
+			content: prompt
+		}]
+	});
+	for (let i = 0; i < turnIndex; i++) {
+		const turn = view.turns[i];
+		if (!turn) continue;
+		messages.push(assistantMessageFromTurn(turn));
+		const toolMsg = toolResultsMessage(turn.toolCalls);
+		if (toolMsg) messages.push(toolMsg);
+	}
+	return messages;
+}
+//#endregion
+//#region src/otel/types.ts
+/** OTLP span kinds (enum integers). */
+const SpanKind = {
+	INTERNAL: 1,
+	CLIENT: 2
+};
+/** OTLP status codes. */
+const StatusCode = {
+	UNSET: 0,
+	OK: 1,
+	ERROR: 2
+};
+//#endregion
+//#region src/otel/emitter.ts
+/**
+* TrajectoryView → OTLP JSON export using OpenTelemetry GenAI semantic conventions.
+*
+* Produces an `ExportTraceServiceRequest` suitable for OTLP/HTTP JSON ingestion.
+* Assertions continue to use {@link TrajectoryView} directly; this is export-only.
+*/
+const INSTRUMENTATION_VERSION = "0.1.0";
+/**
+* Map a {@link TrajectoryView} to OTLP trace JSON.
+*
+* Span tree (siblings under `invoke_agent`, not nested):
+* ```
+* invoke_agent
+* ├── chat {model}
+* ├── execute_tool {name}
+* ├── chat {model}
+* └── ...
+* ```
+*/
+function trajectoryToOtlp(view, options = {}) {
+	const agentName = options.agentName ?? "claude-code";
+	const providerName = options.providerName ?? "anthropic";
+	const serviceName = options.serviceName ?? "harness-eval";
+	const scopeName = options.instrumentationScope ?? "@alis-build/harness-eval";
+	const traceId = traceIdFromSession(view.meta.sessionId);
+	const rootSpanId = spanIdFromKey(traceId, "invoke_agent");
+	const durationMs = Math.max(view.usage.durationMs, 1);
+	const endMs = options.endTimeMs ?? Date.now();
+	const startMs = endMs - durationMs;
+	const rootStartNs = msToNs(startMs);
+	const rootEndNs = msToNs(endMs);
+	const spans = [];
+	const timings = buildSpanTimings(view, startMs, endMs);
+	spans.push({
+		traceId,
+		spanId: rootSpanId,
+		name: "invoke_agent",
+		kind: SpanKind.INTERNAL,
+		startTimeUnixNano: rootStartNs,
+		endTimeUnixNano: rootEndNs,
+		attributes: [
+			strAttr("gen_ai.operation.name", "invoke_agent"),
+			strAttr("gen_ai.agent.name", agentName),
+			strAttr("gen_ai.provider.name", providerName),
+			strAttr("gen_ai.conversation.id", view.meta.sessionId),
+			strAttr("gen_ai.request.model", view.meta.model),
+			strAttr("gen_ai.response.model", view.meta.model),
+			intAttr("gen_ai.usage.input_tokens", view.usage.inputTokens),
+			intAttr("gen_ai.usage.output_tokens", view.usage.outputTokens),
+			boolAttr("harness_eval.success", view.success)
+		],
+		status: viewStatus(view)
+	});
+	let opIndex = 0;
+	for (const turn of view.turns) {
+		const chatTiming = timings[opIndex++];
+		const chatSpanId = spanIdFromKey(traceId, `chat:${turn.turnIndex}`);
+		const inputMessages = inputMessagesBeforeTurn(view, turn.turnIndex, options.prompt);
+		const outputMessages = [assistantMessageFromTurn(turn)];
+		spans.push({
+			traceId,
+			spanId: chatSpanId,
+			parentSpanId: rootSpanId,
+			name: `chat ${view.meta.model}`,
+			kind: SpanKind.CLIENT,
+			startTimeUnixNano: chatTiming.startNs,
+			endTimeUnixNano: chatTiming.endNs,
+			attributes: [
+				strAttr("gen_ai.operation.name", "chat"),
+				strAttr("gen_ai.provider.name", providerName),
+				strAttr("gen_ai.request.model", view.meta.model),
+				strAttr("gen_ai.response.model", view.meta.model),
+				...inputMessages.length > 0 ? [jsonAttr("gen_ai.input.messages", inputMessages)] : [],
+				jsonAttr("gen_ai.output.messages", outputMessages),
+				...turn.stopReason ? [jsonAttr("gen_ai.response.finish_reasons", [mapStopReason(turn.stopReason) ?? turn.stopReason])] : []
+			],
+			status: { code: StatusCode.OK }
+		});
+		if (turn.toolCalls.length === 0) continue;
+		const toolTiming = timings[opIndex++];
+		for (const call of turn.toolCalls) {
+			const toolSpanId = spanIdFromKey(traceId, `tool:${call.callId}`);
+			spans.push({
+				traceId,
+				spanId: toolSpanId,
+				parentSpanId: rootSpanId,
+				name: `execute_tool ${call.name}`,
+				kind: SpanKind.INTERNAL,
+				startTimeUnixNano: toolTiming.startNs,
+				endTimeUnixNano: toolTiming.endNs,
+				attributes: [
+					strAttr("gen_ai.operation.name", "execute_tool"),
+					strAttr("gen_ai.provider.name", providerName),
+					strAttr("gen_ai.tool.name", call.name),
+					strAttr("gen_ai.tool.call.id", call.callId),
+					jsonAttr("gen_ai.tool.call.arguments", call.args ?? {}),
+					...call.result !== null ? [jsonAttr("gen_ai.tool.call.result", call.result)] : [],
+					...call.namespace ? [strAttr("harness_eval.tool.namespace", call.namespace)] : [],
+					boolAttr("harness_eval.tool.is_error", call.isError)
+				],
+				status: call.isError ? {
+					code: StatusCode.ERROR,
+					message: "tool reported error"
+				} : { code: StatusCode.OK }
+			});
+		}
+	}
+	return { resourceSpans: [{
+		resource: { attributes: [strAttr("service.name", serviceName), strAttr("gen_ai.agent.name", agentName)] },
+		scopeSpans: [{
+			scope: {
+				name: scopeName,
+				version: INSTRUMENTATION_VERSION
+			},
+			spans
+		}]
+	}] };
+}
+/** Alias for {@link trajectoryToOtlp} — matches implementation plan naming. */
+const emitOtel = trajectoryToOtlp;
+/** Map view success flag to OTLP span status on the root invoke_agent span. */
+function viewStatus(view) {
+	if (view.success) return { code: StatusCode.OK };
+	return {
+		code: StatusCode.ERROR,
+		message: "harness run did not complete successfully"
+	};
+}
+/**
+* Assign synthetic timestamps to chat and tool spans.
+*
+* Stream-json does not carry per-turn wall times, so we divide the session
+* duration evenly across chat/tool slots for OTLP consumers that require
+* start/end times on every span.
+*/
+function buildSpanTimings(view, startMs, endMs) {
+	const slots = [];
+	for (const turn of view.turns) {
+		slots.push("chat");
+		if (turn.toolCalls.length > 0) slots.push("tools");
+	}
+	if (slots.length === 0) return [];
+	const slotMs = Math.max(endMs - startMs, 1) / slots.length;
+	const timings = [];
+	let offset = startMs;
+	for (const slot of slots) {
+		const slotStart = offset;
+		const slotEnd = offset + slotMs;
+		timings.push({
+			startNs: msToNs(slotStart),
+			endNs: msToNs(slotEnd)
+		});
+		offset = slotEnd;
+	}
+	return timings;
+}
+/**
+* Derive a deterministic 128-bit trace id from the harness session id.
+*
+* Uses SHA-256 truncation so the same session always maps to the same trace.
+*/
+function traceIdFromSession(sessionId) {
+	return createHash("sha256").update(`harness-eval:trace:${sessionId}`).digest("hex").slice(0, 32).toUpperCase();
+}
+/**
+* Derive a deterministic 64-bit span id from trace id and a logical span key.
+*/
+function spanIdFromKey(traceId, key) {
+	return createHash("sha256").update(`${traceId}:span:${key}`).digest("hex").slice(0, 16).toUpperCase();
+}
+/** Convert milliseconds since epoch to OTLP nanosecond timestamp string. */
+function msToNs(ms) {
+	return String(Math.round(ms * 1e6));
+}
+//#endregion
+//#region src/reporter/format-console.ts
+const RESET = "\x1B[0m";
+const GREEN = "\x1B[32m";
+const RED = "\x1B[31m";
+const YELLOW = "\x1B[33m";
+/**
+* Render renderable rows as ANSI-colored console output.
+*
+* @param color When false, emit plain text without escape codes.
+*/
+function formatConsole(rows, color = true) {
+	const lines = [];
+	for (const row of rows) {
+		const status = row.passed ? color ? `${GREEN}PASS${RESET}` : "PASS" : color ? `${RED}FAIL${RESET}` : "FAIL";
+		const crashNote = row.adapterErrors > 0 ? ` ${color ? YELLOW : ""}[${row.adapterErrors} adapter errors]${color ? RESET : ""}` : "";
+		lines.push(`${row.caseId} @ ${row.cellLabel}  ${status}${crashNote}`);
+		if (row.category) lines.push(`  category: ${row.category}`);
+		for (const stat of row.stats) {
+			const marker = stat.meetsThreshold ? color ? `${GREEN}✓${RESET}` : "✓" : color ? `${RED}✗${RESET}` : "✗";
+			const rateStr = formatRate$1(stat);
+			const thresholdPct = (stat.threshold * 100).toFixed(0);
+			let line = `  ├─ ${stat.description}: ${rateStr} [threshold ${thresholdPct}%] ${marker}`;
+			if (stat.delta !== void 0 && stat.baselinePassRate !== void 0) {
+				const arrow = stat.delta >= 0 ? "↑" : "↓";
+				const basePct = (stat.baselinePassRate * 100).toFixed(0);
+				const curPct = (stat.passRate * 100).toFixed(0);
+				const deltaPct = (stat.delta * 100).toFixed(0);
+				line += `  (${basePct}% → ${curPct}% (${arrow}${deltaPct}%))`;
+			}
+			lines.push(line);
+		}
+		lines.push("");
+	}
+	return lines.join("\n").trimEnd();
+}
+/** Format pass rate for display, noting when all reps crashed. */
+function formatRate$1(stat) {
+	if (stat.evaluatedCount === 0) return `0/${stat.totalReps} (all reps crashed)`;
+	const pct = (stat.passRate * 100).toFixed(0);
+	return `${stat.passedCount}/${stat.evaluatedCount} (${pct}%)`;
+}
+//#endregion
+//#region src/reporter/format-json.ts
+/**
+* Serialize a suite report as indented JSON (no transformation).
+*
+* Used by `--format json` and `--output` persistence.
+*/
+function formatJson(report) {
+	return JSON.stringify(report, null, 2);
+}
+//#endregion
+//#region src/reporter/format-markdown.ts
+/** Render renderable rows as a GitHub-flavored markdown report. */
+function formatMarkdown(rows) {
+	const lines = ["# Harness Eval Report", ""];
+	for (const row of rows) {
+		const status = row.passed ? "PASS" : "FAIL";
+		const crashNote = row.adapterErrors > 0 ? ` (${row.adapterErrors} adapter errors)` : "";
+		lines.push(`## ${row.caseId} @ ${row.cellLabel} — ${status}${crashNote}`);
+		if (row.category) lines.push(`**Category:** ${row.category}`);
+		if (row.notes) lines.push("<details><summary>Notes</summary>", row.notes, "</details>");
+		lines.push("");
+		lines.push("| Assertion | Result | Threshold | Status |");
+		lines.push("| --- | --- | --- | --- |");
+		for (const stat of row.stats) {
+			const rateStr = formatRate(stat);
+			const threshold = `${(stat.threshold * 100).toFixed(0)}%`;
+			const statusCell = stat.meetsThreshold ? "✓" : "✗";
+			let result = rateStr;
+			if (stat.delta !== void 0 && stat.baselinePassRate !== void 0) {
+				const base = (stat.baselinePassRate * 100).toFixed(0);
+				const cur = (stat.passRate * 100).toFixed(0);
+				const d = (stat.delta * 100).toFixed(0);
+				const sign = stat.delta >= 0 ? "+" : "";
+				result += ` (${base}% → ${cur}%, ${sign}${d}%)`;
+			}
+			lines.push(`| ${stat.description} | ${result} | ${threshold} | ${statusCell} |`);
+		}
+		lines.push("");
+	}
+	return lines.join("\n").trimEnd();
+}
+/** Format pass rate for markdown tables, noting when all reps crashed. */
+function formatRate(stat) {
+	if (stat.evaluatedCount === 0) return `0/${stat.totalReps} (all reps crashed)`;
+	const pct = (stat.passRate * 100).toFixed(0);
+	return `${stat.passedCount}/${stat.evaluatedCount} (${pct}%)`;
+}
+//#endregion
+//#region src/reporter/renderable.ts
+/** Map a suite report to formatter-ready rows (one per cell). */
+function toRenderableRows(report) {
+	return report.cells.map((cell) => cellToRow(cell));
+}
+/**
+* Attach baseline pass-rate deltas to matching rows.
+*
+* Rows without a matching baseline cell are returned unchanged.
+*/
+function applyBaseline(rows, baseline) {
+	const baselineMap = new Map(baseline.cells.map((c) => [`${c.caseId}::${c.cell.label}`, c]));
+	return rows.map((row) => {
+		const baseCell = baselineMap.get(`${row.caseId}::${row.cellLabel}`);
+		if (!baseCell) return row;
+		const stats = row.stats.map((stat, i) => {
+			const baseStat = baseCell.assertionStats[i];
+			if (!baseStat) return stat;
+			const delta = stat.passRate - baseStat.passRate;
+			return {
+				...stat,
+				baselinePassRate: baseStat.passRate,
+				delta
+			};
+		});
+		return {
+			...row,
+			stats
+		};
+	});
+}
+/** Convert one {@link CellReport} to a {@link RenderableRow}. */
+function cellToRow(cell) {
+	const totalReps = cell.repetitions.length;
+	const stats = cell.assertionStats.map((s) => ({
+		description: s.description,
+		threshold: s.threshold,
+		passedCount: s.passedCount,
+		evaluatedCount: s.evaluatedCount,
+		totalReps,
+		adapterErrors: cell.adapterErrors,
+		passRate: s.passRate,
+		meetsThreshold: s.meetsThreshold
+	}));
+	return {
+		caseId: cell.caseId,
+		category: cell.category,
+		notes: cell.notes,
+		cellLabel: cell.cell.label,
+		passed: cell.passed,
+		adapterErrors: cell.adapterErrors,
+		totalReps,
+		stats
+	};
+}
+//#endregion
+//#region src/reporter/index.ts
+/**
+* Format a {@link SuiteReport} for console, markdown, or JSON output.
+*
+* JSON format bypasses the renderable intermediate model and serializes the
+* report directly. Console and markdown apply optional baseline deltas.
+*/
+function formatReport(report, options) {
+	if (options.format === "json") return formatJson(report);
+	let rows = toRenderableRows(report);
+	if (options.baseline) rows = applyBaseline(rows, options.baseline);
+	const useColor = options.color ?? options.format === "console";
+	if (options.format === "markdown") return formatMarkdown(rows);
+	return formatConsole(rows, useColor);
+}
+//#endregion
+export { serializeToolInput as A, TRAJECTORY_SCHEMA_VERSION as B, trajectoryExactMatch as C, trajectorySingleToolUse as D, trajectoryRecall as E, loadSuiteReport as F, trajectoryToTranscript as I, createCodexGrader as L, gradingReportPassed as M, resolveGradeOptions as N, toEvaluationInstance as O, gradeReport as P, createClaudeGrader as R, trajectoryAnyOrderMatch as S, trajectoryPrecision as T, buildEvalRunEnvelopeFromFiles as _, envelopeCommand as a, computeTrajectoryMetrics as b, getOptionInt as c, resolveGradingArtifactFromSuite as d, resolvePipelineInputs as f, buildEvalRunEnvelope as g, toTrajectory as h, runPipeline as i, formatGradingConsole as j, toTrajectoryInstances as k, hasOption as l, toInstancesJsonl as m, emitOtel as n, parseEnvelopeProjection as o, suiteDirectoryFromPath as p, trajectoryToOtlp as r, getOption as s, formatReport as t, parseArgs as u, enrichRepetitionWithProtojson as v, trajectoryInOrderMatch as w, parseToolInput as x, toHarnessMetrics as y, EVAL_RUN_SCHEMA_VERSION as z };
+
+//# sourceMappingURL=reporter-Biy-5-9M.js.map