@alis-build/harness-eval 0.1.0

package/dist/build-DsVJ_UeU.js

@@ -0,0 +1,1396 @@
+import { i as buildJudgeArgs } from "./claude-code-ycT0JQZF.js";
+import { n as createLimit } from "./suite-chj0j22j.js";
+import { spawn } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { parse } from "yaml";
+import { createHash, randomUUID } from "node:crypto";
+//#region src/types/eval-record.ts
+/** Schema version for {@link EvalRunEnvelope} JSON documents. */
+const EVAL_RUN_SCHEMA_VERSION = "1.0";
+/** Schema version embedded in each {@link TrajectoryView} at export time. */
+const TRAJECTORY_SCHEMA_VERSION = "1.0";
+//#endregion
+//#region src/otel/attributes.ts
+function strAttr(key, value) {
+	return {
+		key,
+		value: { stringValue: value }
+	};
+}
+function intAttr(key, value) {
+	return {
+		key,
+		value: { intValue: String(value) }
+	};
+}
+function boolAttr(key, value) {
+	return {
+		key,
+		value: { boolValue: value }
+	};
+}
+function jsonAttr(key, value) {
+	return {
+		key,
+		value: { stringValue: JSON.stringify(value) }
+	};
+}
+//#endregion
+//#region src/otel/messages.ts
+function mapStopReason(reason) {
+	if (!reason) return void 0;
+	switch (reason) {
+		case "end_turn": return "stop";
+		case "tool_use": return "tool_calls";
+		case "max_tokens": return "length";
+		case "stop_sequence": return "stop";
+		default: return reason;
+	}
+}
+function toolCallPart(call) {
+	return {
+		type: "tool_call",
+		id: call.callId,
+		name: call.name,
+		arguments: call.args ?? {}
+	};
+}
+function toolResponsePart(call) {
+	return {
+		type: "tool_call_response",
+		id: call.callId,
+		result: call.result
+	};
+}
+function assistantMessageFromTurn(turn) {
+	const parts = [];
+	if (turn.text) parts.push({
+		type: "text",
+		content: turn.text
+	});
+	for (const call of turn.toolCalls) parts.push(toolCallPart(call));
+	const finish = mapStopReason(turn.stopReason);
+	return {
+		role: "assistant",
+		parts,
+		...finish ? { finish_reason: finish } : {}
+	};
+}
+function toolResultsMessage(calls) {
+	const parts = calls.filter((c) => c.result !== null).map((c) => toolResponsePart(c));
+	if (parts.length === 0) return null;
+	return {
+		role: "tool",
+		parts
+	};
+}
+/**
+* Input history before the assistant turn at `turnIndex`.
+*/
+function inputMessagesBeforeTurn(view, turnIndex, prompt) {
+	const messages = [];
+	if (prompt) messages.push({
+		role: "user",
+		parts: [{
+			type: "text",
+			content: prompt
+		}]
+	});
+	for (let i = 0; i < turnIndex; i++) {
+		const turn = view.turns[i];
+		if (!turn) continue;
+		messages.push(assistantMessageFromTurn(turn));
+		const toolMsg = toolResultsMessage(turn.toolCalls);
+		if (toolMsg) messages.push(toolMsg);
+	}
+	return messages;
+}
+//#endregion
+//#region src/otel/types.ts
+/** OTLP span kinds (enum integers). */
+const SpanKind = {
+	INTERNAL: 1,
+	CLIENT: 2
+};
+/** OTLP status codes. */
+const StatusCode = {
+	UNSET: 0,
+	OK: 1,
+	ERROR: 2
+};
+//#endregion
+//#region src/otel/emitter.ts
+/**
+* TrajectoryView → OTLP JSON export using OpenTelemetry GenAI semantic conventions.
+*
+* Produces an `ExportTraceServiceRequest` suitable for OTLP/HTTP JSON ingestion.
+* Assertions continue to use {@link TrajectoryView} directly; this is export-only.
+*/
+const INSTRUMENTATION_VERSION = "0.1.0";
+/**
+* Map a {@link TrajectoryView} to OTLP trace JSON.
+*
+* Span tree (siblings under `invoke_agent`, not nested):
+* ```
+* invoke_agent
+* ├── chat {model}
+* ├── execute_tool {name}
+* ├── chat {model}
+* └── ...
+* ```
+*/
+function trajectoryToOtlp(view, options = {}) {
+	const agentName = options.agentName ?? "claude-code";
+	const providerName = options.providerName ?? "anthropic";
+	const serviceName = options.serviceName ?? "harness-eval";
+	const scopeName = options.instrumentationScope ?? "@alis-build/harness-eval";
+	const traceId = traceIdFromSession(view.meta.sessionId);
+	const rootSpanId = spanIdFromKey(traceId, "invoke_agent");
+	const durationMs = Math.max(view.usage.durationMs, 1);
+	const endMs = options.endTimeMs ?? Date.now();
+	const startMs = endMs - durationMs;
+	const rootStartNs = msToNs(startMs);
+	const rootEndNs = msToNs(endMs);
+	const spans = [];
+	const timings = buildSpanTimings(view, startMs, endMs);
+	spans.push({
+		traceId,
+		spanId: rootSpanId,
+		name: "invoke_agent",
+		kind: SpanKind.INTERNAL,
+		startTimeUnixNano: rootStartNs,
+		endTimeUnixNano: rootEndNs,
+		attributes: [
+			strAttr("gen_ai.operation.name", "invoke_agent"),
+			strAttr("gen_ai.agent.name", agentName),
+			strAttr("gen_ai.provider.name", providerName),
+			strAttr("gen_ai.conversation.id", view.meta.sessionId),
+			strAttr("gen_ai.request.model", view.meta.model),
+			strAttr("gen_ai.response.model", view.meta.model),
+			intAttr("gen_ai.usage.input_tokens", view.usage.inputTokens),
+			intAttr("gen_ai.usage.output_tokens", view.usage.outputTokens),
+			boolAttr("harness_eval.success", view.success)
+		],
+		status: viewStatus(view)
+	});
+	let opIndex = 0;
+	for (const turn of view.turns) {
+		const chatTiming = timings[opIndex++];
+		const chatSpanId = spanIdFromKey(traceId, `chat:${turn.turnIndex}`);
+		const inputMessages = inputMessagesBeforeTurn(view, turn.turnIndex, options.prompt);
+		const outputMessages = [assistantMessageFromTurn(turn)];
+		spans.push({
+			traceId,
+			spanId: chatSpanId,
+			parentSpanId: rootSpanId,
+			name: `chat ${view.meta.model}`,
+			kind: SpanKind.CLIENT,
+			startTimeUnixNano: chatTiming.startNs,
+			endTimeUnixNano: chatTiming.endNs,
+			attributes: [
+				strAttr("gen_ai.operation.name", "chat"),
+				strAttr("gen_ai.provider.name", providerName),
+				strAttr("gen_ai.request.model", view.meta.model),
+				strAttr("gen_ai.response.model", view.meta.model),
+				...inputMessages.length > 0 ? [jsonAttr("gen_ai.input.messages", inputMessages)] : [],
+				jsonAttr("gen_ai.output.messages", outputMessages),
+				...turn.stopReason ? [jsonAttr("gen_ai.response.finish_reasons", [mapStopReason(turn.stopReason) ?? turn.stopReason])] : []
+			],
+			status: { code: StatusCode.OK }
+		});
+		if (turn.toolCalls.length === 0) continue;
+		const toolTiming = timings[opIndex++];
+		for (const call of turn.toolCalls) {
+			const toolSpanId = spanIdFromKey(traceId, `tool:${call.callId}`);
+			spans.push({
+				traceId,
+				spanId: toolSpanId,
+				parentSpanId: rootSpanId,
+				name: `execute_tool ${call.name}`,
+				kind: SpanKind.INTERNAL,
+				startTimeUnixNano: toolTiming.startNs,
+				endTimeUnixNano: toolTiming.endNs,
+				attributes: [
+					strAttr("gen_ai.operation.name", "execute_tool"),
+					strAttr("gen_ai.provider.name", providerName),
+					strAttr("gen_ai.tool.name", call.name),
+					strAttr("gen_ai.tool.call.id", call.callId),
+					jsonAttr("gen_ai.tool.call.arguments", call.args ?? {}),
+					...call.result !== null ? [jsonAttr("gen_ai.tool.call.result", call.result)] : [],
+					...call.namespace ? [strAttr("harness_eval.tool.namespace", call.namespace)] : [],
+					boolAttr("harness_eval.tool.is_error", call.isError)
+				],
+				status: call.isError ? {
+					code: StatusCode.ERROR,
+					message: "tool reported error"
+				} : { code: StatusCode.OK }
+			});
+		}
+	}
+	return { resourceSpans: [{
+		resource: { attributes: [strAttr("service.name", serviceName), strAttr("gen_ai.agent.name", agentName)] },
+		scopeSpans: [{
+			scope: {
+				name: scopeName,
+				version: INSTRUMENTATION_VERSION
+			},
+			spans
+		}]
+	}] };
+}
+/** Alias matching the implementation plan naming. */
+const emitOtel = trajectoryToOtlp;
+function viewStatus(view) {
+	if (view.success) return { code: StatusCode.OK };
+	return {
+		code: StatusCode.ERROR,
+		message: "harness run did not complete successfully"
+	};
+}
+function buildSpanTimings(view, startMs, endMs) {
+	const slots = [];
+	for (const turn of view.turns) {
+		slots.push("chat");
+		if (turn.toolCalls.length > 0) slots.push("tools");
+	}
+	if (slots.length === 0) return [];
+	const slotMs = Math.max(endMs - startMs, 1) / slots.length;
+	const timings = [];
+	let offset = startMs;
+	for (const slot of slots) {
+		const slotStart = offset;
+		const slotEnd = offset + slotMs;
+		timings.push({
+			startNs: msToNs(slotStart),
+			endNs: msToNs(slotEnd)
+		});
+		offset = slotEnd;
+	}
+	return timings;
+}
+function traceIdFromSession(sessionId) {
+	return createHash("sha256").update(`harness-eval:trace:${sessionId}`).digest("hex").slice(0, 32).toUpperCase();
+}
+function spanIdFromKey(traceId, key) {
+	return createHash("sha256").update(`${traceId}:span:${key}`).digest("hex").slice(0, 16).toUpperCase();
+}
+function msToNs(ms) {
+	return String(Math.round(ms * 1e6));
+}
+//#endregion
+//#region src/grader/prompt.ts
+function buildGraderPrompt(input) {
+	const expectationList = input.expectations.map((e, i) => `${i + 1}. ${e}`).join("\n");
+	return `${input.systemInstruction ? `${input.systemInstruction.trim()}\n\n` : ""}You are an automated evaluation grader (not the agent under test). Your only job is to score expectations against the transcript below.
+
+Your job is to evaluate each expectation against the transcript and final response.
+PASS only when there is clear evidence in the transcript or final response.
+When uncertain, FAIL — burden of proof is on PASS.
+
+Also critique the expectations themselves if any are trivially satisfied or miss important outcomes.
+
+## Eval prompt
+
+${input.prompt}
+
+## Execution transcript
+
+${input.transcript}
+
+## Expectations to grade
+
+${expectationList}
+
+## Output format
+
+Respond with ONLY a single JSON object (no markdown fences, no commentary) matching this schema:
+
+{
+  "expectations": [
+    { "text": "<original expectation>", "passed": true|false, "evidence": "<quote or description>" }
+  ],
+  "summary": { "passed": <int>, "failed": <int>, "total": <int>, "pass_rate": <0.0-1.0> },
+  "eval_feedback": {
+    "suggestions": [{ "assertion": "<optional>", "reason": "<string>" }],
+    "overall": "<brief assessment>"
+  }
+}
+
+Include every expectation in the same order. summary must match the expectations array.`;
+}
+//#endregion
+//#region src/grader/parse.ts
+function extractClaudeResponseText(stdout) {
+	const trimmed = stdout.trim();
+	if (!trimmed) return "";
+	try {
+		const data = JSON.parse(trimmed);
+		if (Array.isArray(data)) return extractFromEventArray(data) ?? trimmed;
+		if (typeof data === "object" && data !== null) {
+			const event = data;
+			if (event.type === "result" && typeof event.result === "string") return event.result;
+			if (event.type === "assistant" && event.message) {
+				const text = textFromAssistantMessage(event.message);
+				if (text) return text;
+			}
+		}
+	} catch {}
+	return trimmed;
+}
+function extractFromEventArray(events) {
+	const result = events.find((e) => typeof e === "object" && e !== null && e.type === "result");
+	if (result?.result) return result.result;
+	const assistantTexts = [];
+	for (const event of events) if (typeof event === "object" && event !== null && event.type === "assistant") {
+		const text = textFromAssistantMessage(event.message);
+		if (text) assistantTexts.push(text);
+	}
+	if (assistantTexts.length > 0) return assistantTexts[assistantTexts.length - 1];
+	return null;
+}
+function textFromAssistantMessage(message) {
+	if (!message || typeof message !== "object") return null;
+	const content = message.content;
+	if (typeof content === "string") return content;
+	if (!Array.isArray(content)) return null;
+	const texts = [];
+	for (const block of content) if (typeof block === "object" && block !== null && block.type === "text" && typeof block.text === "string") texts.push(block.text);
+	return texts.length > 0 ? texts.join("\n") : null;
+}
+function parseGraderJson(text) {
+	const candidates = [text.trim(), extractJsonBlock(text)];
+	for (const candidate of candidates) {
+		if (!candidate) continue;
+		try {
+			const normalized = normalizeGraderJson(JSON.parse(candidate));
+			if (normalized.expectations.length > 0) return normalized;
+		} catch {
+			continue;
+		}
+	}
+	return null;
+}
+function extractJsonBlock(text) {
+	const fence = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+	if (fence?.[1]) return fence[1].trim();
+	const start = text.indexOf("{");
+	const end = text.lastIndexOf("}");
+	if (start >= 0 && end > start) return text.slice(start, end + 1);
+	return null;
+}
+function normalizeGraderJson(raw) {
+	const expectations = (raw.expectations ?? []).map((e) => ({
+		text: e.text ?? "",
+		passed: Boolean(e.passed),
+		evidence: e.evidence ?? ""
+	}));
+	const passed = expectations.filter((e) => e.passed).length;
+	const failed = expectations.length - passed;
+	const total = expectations.length;
+	const passRate = raw.summary?.pass_rate ?? raw.summary?.passRate ?? (total === 0 ? 0 : passed / total);
+	const summary = {
+		passed: raw.summary?.passed ?? passed,
+		failed: raw.summary?.failed ?? failed,
+		total: raw.summary?.total ?? total,
+		passRate
+	};
+	let evalFeedback;
+	if (raw.eval_feedback) evalFeedback = {
+		suggestions: (raw.eval_feedback.suggestions ?? []).map((s) => ({
+			assertion: s.assertion,
+			reason: s.reason ?? ""
+		})),
+		overall: raw.eval_feedback.overall ?? ""
+	};
+	return {
+		expectations,
+		summary,
+		evalFeedback
+	};
+}
+//#endregion
+//#region src/grader/claude-grader.ts
+/**
+* Grade expectations by spawning Claude as judge (skill-creator grader pattern).
+*/
+const DEFAULT_TIMEOUT_MS = 3e5;
+/**
+* Judge subprocess defaults — grading is a single-shot JSON response, not an agent session.
+* Without these, Claude Code may load plugins/MCP and loop on tools until timeout.
+*/
+const JUDGE_CLAUDE_DEFAULTS = {
+	maxTurns: 1,
+	bare: true,
+	disableSlashCommands: true,
+	noSessionPersistence: true
+};
+function mergeJudgeClaudeOptions(claudeCode) {
+	return {
+		...JUDGE_CLAUDE_DEFAULTS,
+		...claudeCode
+	};
+}
+function createClaudeGrader(options = {}) {
+	return (input) => runClaudeGrader(input, options);
+}
+async function runClaudeGrader(input, options = {}) {
+	const binary = options.binary ?? options.claudeCode?.binary ?? "claude";
+	const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+	const prompt = buildGraderPrompt(input);
+	const model = options.model ?? options.claudeCode?.model;
+	const responseText = extractClaudeResponseText(await spawnCollectStdout(binary, buildJudgeArgs(prompt, {
+		...mergeJudgeClaudeOptions(options.claudeCode),
+		model
+	}), timeoutMs, options.env, options.cwd));
+	const parsed = parseGraderJson(responseText);
+	if (!parsed) return {
+		expectations: input.expectations.map((text) => ({
+			text,
+			passed: false,
+			evidence: "Grader returned unparseable output"
+		})),
+		summary: {
+			passed: 0,
+			failed: input.expectations.length,
+			total: input.expectations.length,
+			passRate: 0
+		},
+		error: `failed to parse grader JSON from response: ${responseText.slice(0, 200)}`
+	};
+	const expectations = input.expectations.map((text, i) => {
+		const graded = parsed.expectations[i];
+		return {
+			text,
+			passed: graded?.passed ?? false,
+			evidence: graded?.evidence ?? "No evidence returned"
+		};
+	});
+	const passed = expectations.filter((e) => e.passed).length;
+	const total = expectations.length;
+	return {
+		expectations,
+		summary: {
+			passed,
+			failed: total - passed,
+			total,
+			passRate: total === 0 ? 0 : passed / total
+		},
+		evalFeedback: parsed.evalFeedback
+	};
+}
+function spawnCollectStdout(binary, args, timeoutMs, extraEnv, cwd) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(binary, args, {
+			env: buildChildEnv(extraEnv),
+			cwd,
+			stdio: [
+				"ignore",
+				"pipe",
+				"pipe"
+			]
+		});
+		const chunks = [];
+		child.stdout?.setEncoding("utf8");
+		child.stdout?.on("data", (c) => chunks.push(c));
+		const stderrChunks = [];
+		child.stderr?.setEncoding("utf8");
+		child.stderr?.on("data", (c) => stderrChunks.push(c));
+		const timer = setTimeout(() => {
+			child.kill("SIGTERM");
+			const stderrHint = stderrChunks.join("").trim().slice(0, 400);
+			reject(/* @__PURE__ */ new Error(`grader timed out after ${timeoutMs}ms` + (stderrHint ? ` (stderr: ${stderrHint})` : "")));
+		}, timeoutMs);
+		const finalize = (err) => {
+			clearTimeout(timer);
+			if (err) reject(err);
+			else resolve(chunks.join(""));
+		};
+		child.on("error", (err) => finalize(err));
+		child.on("close", (code) => {
+			if (code !== 0 && chunks.length === 0) finalize(/* @__PURE__ */ new Error(`grader exited ${code}: ${stderrChunks.join("").slice(0, 500)}`));
+			else finalize();
+		});
+	});
+}
+function buildChildEnv(extraEnv) {
+	const env = {
+		...process.env,
+		...extraEnv
+	};
+	delete env.CLAUDECODE;
+	return env;
+}
+//#endregion
+//#region src/grader/expectations.ts
+/**
+* Load expectations sidecar (YAML or JSON).
+*/
+async function loadExpectationsMap(path) {
+	const text = await readFile(path, "utf8");
+	const trimmed = path.trim().toLowerCase();
+	let raw;
+	if (trimmed.endsWith(".json")) raw = JSON.parse(text);
+	else raw = parse(text);
+	if (!raw || typeof raw !== "object") throw new Error(`expectations file must be an object mapping case ids to lists`);
+	const map = {};
+	for (const [caseId, value] of Object.entries(raw)) {
+		if (!Array.isArray(value)) throw new Error(`expectations for case "${caseId}" must be an array of strings`);
+		map[caseId] = value.map(String);
+	}
+	return map;
+}
+//#endregion
+//#region src/grader/transcript.ts
+const MAX_RESULT_CHARS = 4e3;
+function trajectoryToTranscript(view, prompt) {
+	const lines = [];
+	if (prompt) lines.push("## User prompt", "", prompt, "");
+	for (const turn of view.turns) {
+		lines.push(`## Assistant turn ${turn.turnIndex + 1}`, "");
+		if (turn.text) lines.push(turn.text, "");
+		for (const call of turn.toolCalls) {
+			lines.push(`[Tool call] ${call.name} (id=${call.callId})`);
+			lines.push(`Arguments: ${formatJson$1(call.args)}`);
+			if (call.result !== null) {
+				lines.push(`[Tool result] ${formatResult(call.result)}`);
+				if (call.isError) lines.push("(tool reported error)");
+			} else lines.push("[Tool result] (none observed)");
+			lines.push("");
+		}
+		if (turn.stopReason) lines.push(`Stop reason: ${turn.stopReason}`, "");
+	}
+	const finalInTurns = view.turns.some((t) => t.text === view.finalResponse);
+	if (view.finalResponse && !finalInTurns) lines.push("## Final response", "", view.finalResponse, "");
+	lines.push("## Session metadata", `session_id: ${view.meta.sessionId}`, `model: ${view.meta.model}`, `cwd: ${view.meta.cwd}`, `success: ${view.success}`, `tool_calls: ${view.toolCalls.length}`, `duration_ms: ${view.usage.durationMs}`, `input_tokens: ${view.usage.inputTokens}`, `output_tokens: ${view.usage.outputTokens}`);
+	return lines.join("\n").trimEnd();
+}
+function formatJson$1(value) {
+	try {
+		return JSON.stringify(value);
+	} catch {
+		return String(value);
+	}
+}
+function formatResult(result) {
+	if (typeof result === "string") return truncate(result);
+	return truncate(formatJson$1(result));
+}
+function truncate(text) {
+	if (text.length <= MAX_RESULT_CHARS) return text;
+	return `${text.slice(0, MAX_RESULT_CHARS)}… (truncated)`;
+}
+//#endregion
+//#region src/grader/grade-report.ts
+/**
+* Grade a harness-eval SuiteReport with outcome expectations (LLM judge).
+*/
+async function gradeReport(report, options = {}) {
+	const expectationsMap = options.expectationsPath ? await loadExpectationsMap(options.expectationsPath) : {};
+	const gradeFn = options.gradeFn ?? createClaudeGrader({
+		binary: options.binary,
+		model: options.model,
+		timeoutMs: options.timeoutMs,
+		env: options.env,
+		cwd: options.cwd,
+		claudeCode: options.claudeCode
+	});
+	const limit = createLimit(options.maxConcurrent ?? 2);
+	const tasks = [];
+	for (const cell of report.cells) {
+		const expectations = cell.expectations ?? expectationsMap[cell.caseId] ?? [];
+		if (expectations.length === 0) continue;
+		for (const rep of cell.repetitions) {
+			if (!rep.adapterResult) continue;
+			tasks.push({
+				cell,
+				rep,
+				expectations
+			});
+		}
+	}
+	const gradeStartTs = Date.now();
+	options.onProgress?.({
+		kind: "grade-start",
+		total: tasks.length
+	});
+	const results = await Promise.all(tasks.map(({ cell, rep, expectations }) => limit(async () => {
+		const start = Date.now();
+		const view = rep.adapterResult.view;
+		const prompt = cell.prompt ?? "";
+		const transcript = trajectoryToTranscript(view, prompt);
+		try {
+			const graded = await gradeFn({
+				prompt,
+				transcript,
+				expectations,
+				systemInstruction: options.systemInstruction
+			});
+			const result = {
+				caseId: cell.caseId,
+				cellLabel: cell.cell.label,
+				repetitionIndex: rep.repetitionIndex,
+				prompt,
+				expectations: graded.expectations,
+				summary: graded.summary,
+				evalFeedback: graded.evalFeedback,
+				graderError: graded.error,
+				durationMs: Date.now() - start
+			};
+			options.onProgress?.({
+				kind: "grade-complete",
+				caseId: result.caseId,
+				cellLabel: result.cellLabel,
+				repetitionIndex: result.repetitionIndex,
+				passed: result.summary.passed,
+				failed: result.summary.failed,
+				durationMs: result.durationMs,
+				graderError: result.graderError
+			});
+			return result;
+		} catch (err) {
+			const message = err instanceof Error ? err.message : String(err);
+			const result = {
+				caseId: cell.caseId,
+				cellLabel: cell.cell.label,
+				repetitionIndex: rep.repetitionIndex,
+				prompt,
+				expectations: expectations.map((text) => ({
+					text,
+					passed: false,
+					evidence: message
+				})),
+				summary: {
+					passed: 0,
+					failed: expectations.length,
+					total: expectations.length,
+					passRate: 0
+				},
+				graderError: message,
+				durationMs: Date.now() - start
+			};
+			options.onProgress?.({
+				kind: "grade-complete",
+				caseId: result.caseId,
+				cellLabel: result.cellLabel,
+				repetitionIndex: result.repetitionIndex,
+				passed: 0,
+				failed: expectations.length,
+				durationMs: result.durationMs,
+				graderError: message
+			});
+			return result;
+		}
+	})));
+	results.sort((a, b) => {
+		const keyA = `${a.caseId}::${a.cellLabel}::${a.repetitionIndex}`;
+		const keyB = `${b.caseId}::${b.cellLabel}::${b.repetitionIndex}`;
+		return keyA.localeCompare(keyB);
+	});
+	const totalExpectations = results.reduce((n, r) => n + r.summary.total, 0);
+	const passedExpectations = results.reduce((n, r) => n + r.summary.passed, 0);
+	options.onProgress?.({
+		kind: "grade-done",
+		durationMs: Date.now() - gradeStartTs,
+		totalExpectations,
+		passedExpectations
+	});
+	return {
+		gradedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		sourceReport: options.sourceReport ?? "",
+		gradingConfigPath: options.gradingConfigPath,
+		results,
+		summary: {
+			passed: passedExpectations,
+			failed: totalExpectations - passedExpectations,
+			total: totalExpectations,
+			passRate: totalExpectations === 0 ? 0 : passedExpectations / totalExpectations
+		}
+	};
+}
+async function loadSuiteReport(path) {
+	const text = await readFile(path, "utf8");
+	return JSON.parse(text);
+}
+//#endregion
+//#region src/grader/resolve-grade-options.ts
+/**
+* Merge standalone grading YAML with CLI flags (CLI wins).
+*/
+function resolveGradeOptions(fileConfig, cli = {}, configPath) {
+	const judge = fileConfig?.judge;
+	const adapter = judge?.adapter ?? "claude-code";
+	if (adapter !== "claude-code") throw new Error(`unsupported grading adapter "${adapter}" (only claude-code today)`);
+	const claudeCode = judge?.claudeCode ?? {};
+	const binary = cli.binary ?? claudeCode.binary;
+	const model = cli.model ?? judge?.model ?? claudeCode.model;
+	return {
+		sourceReport: cli.sourceReport,
+		expectationsPath: cli.expectationsPath,
+		model,
+		binary,
+		timeoutMs: cli.timeoutMs ?? judge?.timeoutMs,
+		maxConcurrent: cli.maxConcurrent ?? judge?.maxConcurrent,
+		systemInstruction: judge?.system_instruction,
+		env: judge?.env,
+		cwd: judge?.cwd,
+		claudeCode: {
+			...claudeCode,
+			binary: void 0,
+			model: void 0
+		},
+		gradingConfigPath: configPath
+	};
+}
+//#endregion
+//#region src/grader/format-console.ts
+const RESET$1 = "\x1B[0m";
+const GREEN$1 = "\x1B[32m";
+const RED$1 = "\x1B[31m";
+const DIM = "\x1B[2m";
+function formatGradingConsole(report, color = true) {
+	const lines = [];
+	if (report.results.length === 0) {
+		lines.push("No repetitions graded. Add expectations to the suite YAML or pass --expectations.");
+		return lines.join("\n");
+	}
+	for (const result of report.results) {
+		const status = result.summary.failed === 0 && !result.graderError ? color ? `${GREEN$1}PASS${RESET$1}` : "PASS" : color ? `${RED$1}FAIL${RESET$1}` : "FAIL";
+		lines.push(`${result.caseId} @ ${result.cellLabel} rep${result.repetitionIndex}  ${status}`);
+		if (result.graderError) lines.push(color ? `  ${RED$1}grader error: ${result.graderError}${RESET$1}` : `  grader error: ${result.graderError}`);
+		for (const exp of result.expectations) {
+			const marker = exp.passed ? color ? `${GREEN$1}✓${RESET$1}` : "✓" : color ? `${RED$1}✗${RESET$1}` : "✗";
+			lines.push(`  ├─ ${exp.text} ${marker}`);
+			if (!exp.passed || exp.evidence) lines.push(color ? `  │  ${DIM}${exp.evidence}${RESET$1}` : `  │  ${exp.evidence}`);
+		}
+		const pct = (result.summary.passRate * 100).toFixed(0);
+		lines.push(`  └─ ${result.summary.passed}/${result.summary.total} (${pct}%) expectations`);
+		lines.push("");
+	}
+	const overallPct = (report.summary.passRate * 100).toFixed(0);
+	lines.push(`Overall: ${report.summary.passed}/${report.summary.total} (${overallPct}%) expectations passed`);
+	return lines.join("\n").trimEnd();
+}
+function gradingReportPassed(report) {
+	return report.results.every((r) => !r.graderError && r.summary.failed === 0 && r.summary.total > 0);
+}
+//#endregion
+//#region src/reporter/format-console.ts
+const RESET = "\x1B[0m";
+const GREEN = "\x1B[32m";
+const RED = "\x1B[31m";
+const YELLOW = "\x1B[33m";
+function formatConsole(rows, color = true) {
+	const lines = [];
+	for (const row of rows) {
+		const status = row.passed ? color ? `${GREEN}PASS${RESET}` : "PASS" : color ? `${RED}FAIL${RESET}` : "FAIL";
+		const crashNote = row.adapterErrors > 0 ? ` ${color ? YELLOW : ""}[${row.adapterErrors} adapter errors]${color ? RESET : ""}` : "";
+		lines.push(`${row.caseId} @ ${row.cellLabel}  ${status}${crashNote}`);
+		if (row.category) lines.push(`  category: ${row.category}`);
+		for (const stat of row.stats) {
+			const marker = stat.meetsThreshold ? color ? `${GREEN}✓${RESET}` : "✓" : color ? `${RED}✗${RESET}` : "✗";
+			const rateStr = formatRate$1(stat);
+			const thresholdPct = (stat.threshold * 100).toFixed(0);
+			let line = `  ├─ ${stat.description}: ${rateStr} [threshold ${thresholdPct}%] ${marker}`;
+			if (stat.delta !== void 0 && stat.baselinePassRate !== void 0) {
+				const arrow = stat.delta >= 0 ? "↑" : "↓";
+				const basePct = (stat.baselinePassRate * 100).toFixed(0);
+				const curPct = (stat.passRate * 100).toFixed(0);
+				const deltaPct = (stat.delta * 100).toFixed(0);
+				line += `  (${basePct}% → ${curPct}% (${arrow}${deltaPct}%))`;
+			}
+			lines.push(line);
+		}
+		lines.push("");
+	}
+	return lines.join("\n").trimEnd();
+}
+function formatRate$1(stat) {
+	if (stat.evaluatedCount === 0) return `0/${stat.totalReps} (all reps crashed)`;
+	const pct = (stat.passRate * 100).toFixed(0);
+	return `${stat.passedCount}/${stat.evaluatedCount} (${pct}%)`;
+}
+//#endregion
+//#region src/reporter/format-json.ts
+function formatJson(report) {
+	return JSON.stringify(report, null, 2);
+}
+//#endregion
+//#region src/reporter/format-markdown.ts
+function formatMarkdown(rows) {
+	const lines = ["# Harness Eval Report", ""];
+	for (const row of rows) {
+		const status = row.passed ? "PASS" : "FAIL";
+		const crashNote = row.adapterErrors > 0 ? ` (${row.adapterErrors} adapter errors)` : "";
+		lines.push(`## ${row.caseId} @ ${row.cellLabel} — ${status}${crashNote}`);
+		if (row.category) lines.push(`**Category:** ${row.category}`);
+		if (row.notes) lines.push("<details><summary>Notes</summary>", row.notes, "</details>");
+		lines.push("");
+		lines.push("| Assertion | Result | Threshold | Status |");
+		lines.push("| --- | --- | --- | --- |");
+		for (const stat of row.stats) {
+			const rateStr = formatRate(stat);
+			const threshold = `${(stat.threshold * 100).toFixed(0)}%`;
+			const statusCell = stat.meetsThreshold ? "✓" : "✗";
+			let result = rateStr;
+			if (stat.delta !== void 0 && stat.baselinePassRate !== void 0) {
+				const base = (stat.baselinePassRate * 100).toFixed(0);
+				const cur = (stat.passRate * 100).toFixed(0);
+				const d = (stat.delta * 100).toFixed(0);
+				const sign = stat.delta >= 0 ? "+" : "";
+				result += ` (${base}% → ${cur}%, ${sign}${d}%)`;
+			}
+			lines.push(`| ${stat.description} | ${result} | ${threshold} | ${statusCell} |`);
+		}
+		lines.push("");
+	}
+	return lines.join("\n").trimEnd();
+}
+function formatRate(stat) {
+	if (stat.evaluatedCount === 0) return `0/${stat.totalReps} (all reps crashed)`;
+	const pct = (stat.passRate * 100).toFixed(0);
+	return `${stat.passedCount}/${stat.evaluatedCount} (${pct}%)`;
+}
+//#endregion
+//#region src/reporter/renderable.ts
+function toRenderableRows(report) {
+	return report.cells.map((cell) => cellToRow(cell));
+}
+function applyBaseline(rows, baseline) {
+	const baselineMap = new Map(baseline.cells.map((c) => [`${c.caseId}::${c.cell.label}`, c]));
+	return rows.map((row) => {
+		const baseCell = baselineMap.get(`${row.caseId}::${row.cellLabel}`);
+		if (!baseCell) return row;
+		const stats = row.stats.map((stat, i) => {
+			const baseStat = baseCell.assertionStats[i];
+			if (!baseStat) return stat;
+			const delta = stat.passRate - baseStat.passRate;
+			return {
+				...stat,
+				baselinePassRate: baseStat.passRate,
+				delta
+			};
+		});
+		return {
+			...row,
+			stats
+		};
+	});
+}
+function cellToRow(cell) {
+	const totalReps = cell.repetitions.length;
+	const stats = cell.assertionStats.map((s) => ({
+		description: s.description,
+		threshold: s.threshold,
+		passedCount: s.passedCount,
+		evaluatedCount: s.evaluatedCount,
+		totalReps,
+		adapterErrors: cell.adapterErrors,
+		passRate: s.passRate,
+		meetsThreshold: s.meetsThreshold
+	}));
+	return {
+		caseId: cell.caseId,
+		category: cell.category,
+		notes: cell.notes,
+		cellLabel: cell.cell.label,
+		passed: cell.passed,
+		adapterErrors: cell.adapterErrors,
+		totalReps,
+		stats
+	};
+}
+//#endregion
+//#region src/reporter/index.ts
+function formatReport(report, options) {
+	if (options.format === "json") return formatJson(report);
+	let rows = toRenderableRows(report);
+	if (options.baseline) rows = applyBaseline(rows, options.baseline);
+	const useColor = options.color ?? options.format === "console";
+	if (options.format === "markdown") return formatMarkdown(rows);
+	return formatConsole(rows, useColor);
+}
+//#endregion
+//#region src/eval-interchange/build.ts
+const DEFAULT_AGENT_ID = "agent";
+function serializeToolInput(args) {
+	return JSON.stringify(args ?? {});
+}
+function parseToolInput(toolInput) {
+	try {
+		return JSON.parse(toolInput);
+	} catch {
+		return toolInput;
+	}
+}
+function toolCallToInterchange(toolCall) {
+	return {
+		tool_name: toolCall.name,
+		tool_input: serializeToolInput(toolCall.args)
+	};
+}
+function interchangeToTabular(toolCall) {
+	return {
+		tool_name: toolCall.tool_name,
+		tool_input: parseToolInput(toolCall.tool_input)
+	};
+}
+function predictedTrajectoryFromView(view) {
+	return view.toolCalls.map(toolCallToInterchange);
+}
+function buildAgentTrace(view, agentId = DEFAULT_AGENT_ID) {
+	const agents = { [agentId]: {
+		agent_id: agentId,
+		agent_type: "assistant",
+		description: view.meta.model,
+		tools: view.meta.availableTools.map((name) => ({ name }))
+	} };
+	const activeTools = view.meta.availableTools.map((name) => ({ name }));
+	return {
+		agents,
+		turns: view.turns.map((turn) => {
+			const events = [];
+			if (turn.text) events.push({
+				author: agentId,
+				content: { parts: [{ text: turn.text }] },
+				active_tools: activeTools
+			});
+			for (const toolCall of turn.toolCalls) {
+				events.push({
+					author: agentId,
+					content: { parts: [{ function_call: {
+						name: toolCall.name,
+						args: toolCall.args ?? {}
+					} }] },
+					active_tools: activeTools
+				});
+				if (toolCall.result !== null && toolCall.result !== void 0) events.push({
+					author: agentId,
+					content: { parts: [{ function_response: {
+						name: toolCall.name,
+						response: toolCall.result
+					} }] },
+					active_tools: activeTools
+				});
+			}
+			return {
+				turn_index: turn.turnIndex,
+				events
+			};
+		})
+	};
+}
+function latencyInSeconds(view) {
+	return view.usage.durationMs / 1e3;
+}
+//#endregion
+//#region src/metrics/trajectory.ts
+function normalizeToolCall$1(toolCall) {
+	if (typeof toolCall.tool_input === "string") return {
+		tool_name: toolCall.tool_name,
+		tool_input: toolCall.tool_input
+	};
+	return {
+		tool_name: toolCall.tool_name,
+		tool_input: serializeToolInput(toolCall.tool_input)
+	};
+}
+function normalizeTrajectory(trajectory) {
+	return trajectory.map(normalizeToolCall$1);
+}
+function toolCallKey(toolCall) {
+	return `${toolCall.tool_name}\0${toolCall.tool_input}`;
+}
+function multisetIntersectionSize(predicted, reference) {
+	const refCounts = /* @__PURE__ */ new Map();
+	for (const toolCall of reference) {
+		const key = toolCallKey(toolCall);
+		refCounts.set(key, (refCounts.get(key) ?? 0) + 1);
+	}
+	let matched = 0;
+	for (const toolCall of predicted) {
+		const key = toolCallKey(toolCall);
+		const count = refCounts.get(key) ?? 0;
+		if (count > 0) {
+			matched += 1;
+			refCounts.set(key, count - 1);
+		}
+	}
+	return matched;
+}
+function isSubsequence(predicted, reference) {
+	let refIndex = 0;
+	for (const toolCall of predicted) {
+		if (refIndex >= reference.length) break;
+		if (toolCallKey(toolCall) === toolCallKey(reference[refIndex])) refIndex += 1;
+	}
+	return refIndex === reference.length;
+}
+function arraysEqual(left, right) {
+	if (left.length !== right.length) return false;
+	return left.every((toolCall, index) => {
+		const other = right[index];
+		return toolCallKey(toolCall) === toolCallKey(other);
+	});
+}
+function trajectoryExactMatch(predicted, reference) {
+	return arraysEqual(normalizeTrajectory(predicted), normalizeTrajectory(reference)) ? 1 : 0;
+}
+function trajectoryInOrderMatch(predicted, reference) {
+	return isSubsequence(normalizeTrajectory(predicted), normalizeTrajectory(reference)) ? 1 : 0;
+}
+function trajectoryAnyOrderMatch(predicted, reference) {
+	const predictedNorm = normalizeTrajectory(predicted);
+	const referenceNorm = normalizeTrajectory(reference);
+	if (predictedNorm.length !== referenceNorm.length) return 0;
+	const predictedKeys = predictedNorm.map(toolCallKey).sort();
+	const referenceKeys = referenceNorm.map(toolCallKey).sort();
+	return predictedKeys.every((key, index) => key === referenceKeys[index]) ? 1 : 0;
+}
+function trajectoryPrecision(predicted, reference) {
+	const predictedNorm = normalizeTrajectory(predicted);
+	if (predictedNorm.length === 0) return reference.length === 0 ? 1 : 0;
+	return multisetIntersectionSize(predictedNorm, normalizeTrajectory(reference)) / predictedNorm.length;
+}
+function trajectoryRecall(predicted, reference) {
+	const referenceNorm = normalizeTrajectory(reference);
+	if (referenceNorm.length === 0) return predicted.length === 0 ? 1 : 0;
+	return multisetIntersectionSize(normalizeTrajectory(predicted), referenceNorm) / referenceNorm.length;
+}
+function trajectorySingleToolUse(predicted, reference) {
+	const predictedNorm = normalizeTrajectory(predicted);
+	const referenceNorm = normalizeTrajectory(reference);
+	if (predictedNorm.length !== 1 || referenceNorm.length !== 1) return 0;
+	return toolCallKey(predictedNorm[0]) === toolCallKey(referenceNorm[0]) ? 1 : 0;
+}
+function computeTrajectoryMetrics(predicted, reference) {
+	return {
+		trajectory_exact_match: trajectoryExactMatch(predicted, reference),
+		trajectory_in_order_match: trajectoryInOrderMatch(predicted, reference),
+		trajectory_any_order_match: trajectoryAnyOrderMatch(predicted, reference),
+		trajectory_precision: trajectoryPrecision(predicted, reference),
+		trajectory_recall: trajectoryRecall(predicted, reference),
+		trajectory_single_tool_use: trajectorySingleToolUse(predicted, reference)
+	};
+}
+//#endregion
+//#region src/metrics/tool-calls.ts
+function normalizeToolCall(toolCall) {
+	if (typeof toolCall.tool_input === "string") return {
+		tool_name: toolCall.tool_name,
+		tool_input: toolCall.tool_input
+	};
+	return {
+		tool_name: toolCall.tool_name,
+		tool_input: serializeToolInput(toolCall.tool_input)
+	};
+}
+function parsedArgs(toolCall) {
+	const parsed = parseToolInput(toolCall.tool_input);
+	if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return null;
+	return parsed;
+}
+function toolCallValid(toolCall) {
+	const normalized = normalizeToolCall(toolCall);
+	if (!normalized.tool_name.trim()) return 0;
+	try {
+		JSON.parse(normalized.tool_input);
+		return 1;
+	} catch {
+		return 0;
+	}
+}
+function toolNameMatch(predicted, reference) {
+	const predictedNorm = normalizeToolCall(predicted);
+	const referenceNorm = normalizeToolCall(reference);
+	return predictedNorm.tool_name === referenceNorm.tool_name ? 1 : 0;
+}
+function toolParameterKeyMatch(predicted, reference) {
+	if (toolNameMatch(predicted, reference) === 0) return 0;
+	const predictedArgs = parsedArgs(normalizeToolCall(predicted));
+	const referenceArgs = parsedArgs(normalizeToolCall(reference));
+	if (predictedArgs === null || referenceArgs === null) return 0;
+	const predictedKeys = Object.keys(predictedArgs).sort();
+	const referenceKeys = Object.keys(referenceArgs).sort();
+	if (predictedKeys.length !== referenceKeys.length) return 0;
+	return predictedKeys.every((key, index) => key === referenceKeys[index]) ? 1 : 0;
+}
+function valuesEqual(left, right, useStrictStringMatch) {
+	if (useStrictStringMatch) return JSON.stringify(left) === JSON.stringify(right);
+	return JSON.stringify(left) === JSON.stringify(right);
+}
+function toolParameterKvMatch(predicted, reference, options = {}) {
+	if (toolParameterKeyMatch(predicted, reference) === 0) return 0;
+	const predictedArgs = parsedArgs(normalizeToolCall(predicted));
+	const referenceArgs = parsedArgs(normalizeToolCall(reference));
+	for (const key of Object.keys(referenceArgs)) if (!valuesEqual(predictedArgs[key], referenceArgs[key], options.useStrictStringMatch ?? false)) return 0;
+	return 1;
+}
+function computeToolCallMetrics(predicted, reference, options = {}) {
+	const pairCount = Math.max(predicted.length, reference.length, 1);
+	let valid = 0;
+	let nameMatch = 0;
+	let keyMatch = 0;
+	let kvMatch = 0;
+	for (let index = 0; index < pairCount; index += 1) {
+		const predictedCall = predicted[index];
+		const referenceCall = reference[index];
+		if (!predictedCall) continue;
+		valid += toolCallValid(predictedCall);
+		if (!referenceCall) continue;
+		nameMatch += toolNameMatch(predictedCall, referenceCall);
+		keyMatch += toolParameterKeyMatch(predictedCall, referenceCall);
+		kvMatch += toolParameterKvMatch(predictedCall, referenceCall, options);
+	}
+	return {
+		tool_call_valid: valid / pairCount,
+		tool_name_match: nameMatch / pairCount,
+		tool_parameter_key_match: keyMatch / pairCount,
+		tool_parameter_kv_match: kvMatch / pairCount
+	};
+}
+//#endregion
+//#region src/eval-interchange/projections.ts
+/**
+* Envelope projection methods for eval interchange output.
+*/
+function repetitionInterchangeFields(repetition) {
+	if (!repetition.trajectory) return { predicted_trajectory: [] };
+	return {
+		predicted_trajectory: repetition.predicted_trajectory ?? predictedTrajectoryFromView(repetition.trajectory),
+		agent_trace: repetition.agent_trace ?? buildAgentTrace(repetition.trajectory),
+		latency_in_seconds: repetition.latency_in_seconds ?? latencyInSeconds(repetition.trajectory),
+		failure: repetition.failure ?? (repetition.trajectory.success ? 0 : 1)
+	};
+}
+function referenceTrajectoryForCell(cell) {
+	return cell.reference_trajectory;
+}
+function repetitionToDatasetRow(cell, repetition) {
+	const fields = repetitionInterchangeFields(repetition);
+	if (!repetition.trajectory) return {
+		prompt: cell.prompt,
+		response: void 0,
+		predicted_trajectory: [],
+		reference_trajectory: referenceTrajectoryForCell(cell),
+		latency_in_seconds: repetition.durationMs / 1e3,
+		failure: 1,
+		human_ratings: cell.human_ratings
+	};
+	return {
+		prompt: cell.prompt,
+		response: repetition.trajectory.finalResponse,
+		predicted_trajectory: fields.predicted_trajectory.map(interchangeToTabular),
+		reference_trajectory: referenceTrajectoryForCell(cell),
+		latency_in_seconds: fields.latency_in_seconds ?? repetition.durationMs / 1e3,
+		failure: fields.failure ?? 1,
+		human_ratings: cell.human_ratings
+	};
+}
+function repetitionToProtoInstance(cell, repetition) {
+	const fields = repetitionInterchangeFields(repetition);
+	if (!repetition.trajectory) return null;
+	const reference = referenceTrajectoryForCell(cell);
+	return {
+		prompt: cell.prompt,
+		response: repetition.trajectory.finalResponse,
+		predicted_trajectory: { tool_calls: fields.predicted_trajectory },
+		reference_trajectory: reference ? { tool_calls: reference.map((toolCall) => ({
+			tool_name: toolCall.tool_name,
+			tool_input: typeof toolCall.tool_input === "string" ? toolCall.tool_input : JSON.stringify(toolCall.tool_input ?? {})
+		})) } : void 0
+	};
+}
+function repetitionToAgentTrace(repetition) {
+	return repetitionInterchangeFields(repetition).agent_trace ?? null;
+}
+function computeRepetitionMetrics(repetition, referenceTrajectory) {
+	if (!referenceTrajectory?.length) return {};
+	const predictedTabular = (repetition.predicted_trajectory ?? (repetition.trajectory ? predictedTrajectoryFromView(repetition.trajectory) : [])).map(interchangeToTabular);
+	return {
+		trajectoryMetrics: computeTrajectoryMetrics(predictedTabular, referenceTrajectory),
+		toolCallMetrics: computeToolCallMetrics(predictedTabular, referenceTrajectory)
+	};
+}
+function toTrajectory(envelope) {
+	const rows = [];
+	for (const cell of envelope.cells) for (const repetition of cell.repetitions) {
+		const row = repetitionToDatasetRow(cell, repetition);
+		if (row) rows.push(row);
+	}
+	return rows;
+}
+function toProtoInstances(envelope) {
+	const instances = [];
+	for (const cell of envelope.cells) for (const repetition of cell.repetitions) {
+		const instance = repetitionToProtoInstance(cell, repetition);
+		if (instance) instances.push(instance);
+	}
+	return instances;
+}
+function toAgentTrace(envelope) {
+	const traces = [];
+	for (const cell of envelope.cells) for (const repetition of cell.repetitions) {
+		const trace = repetitionToAgentTrace(repetition);
+		if (trace) traces.push(trace);
+	}
+	return traces;
+}
+function enrichRepetitionWithInterchange(repetition, referenceTrajectory) {
+	if (!repetition.trajectory) return repetition;
+	const predicted_trajectory = predictedTrajectoryFromView(repetition.trajectory);
+	const agent_trace = buildAgentTrace(repetition.trajectory);
+	const latency_in_seconds = latencyInSeconds(repetition.trajectory);
+	const failure = repetition.trajectory.success ? 0 : 1;
+	const metrics = computeRepetitionMetrics({
+		...repetition,
+		predicted_trajectory,
+		agent_trace,
+		latency_in_seconds,
+		failure
+	}, referenceTrajectory);
+	return {
+		...repetition,
+		predicted_trajectory,
+		agent_trace,
+		latency_in_seconds,
+		failure,
+		trajectoryMetrics: metrics.trajectoryMetrics,
+		toolCallMetrics: metrics.toolCallMetrics
+	};
+}
+//#endregion
+//#region src/eval-record/build.ts
+/**
+* Build {@link EvalRunEnvelope} from harness-eval run and grading reports.
+*/
+function extractRawEvents(adapterResult) {
+	if (adapterResult !== null && typeof adapterResult === "object" && "rawEvents" in adapterResult && Array.isArray(adapterResult.rawEvents)) return adapterResult.rawEvents;
+}
+function outcomePassForCell(caseId, cellLabel, repetitions) {
+	const graded = repetitions.filter((r) => r.outcomeGrades);
+	if (graded.length === 0) return void 0;
+	return graded.every((r) => r.outcomeGrades.error === void 0 && r.outcomeGrades.summary.failed === 0);
+}
+/**
+* Convert a {@link SuiteReport} (and optional grading) into a versioned
+* {@link EvalRunEnvelope} for storage or API handoff.
+*/
+function buildEvalRunEnvelope(report, options = {}) {
+	const includeTranscript = options.includeTranscript !== false;
+	const includeRaw = options.includeRawStreamEvents === true;
+	const judge = options.grading?.judge ?? { id: "harness-eval/claude-grader" };
+	const cells = report.cells.map((cell) => {
+		const prompt = cell.prompt ?? "";
+		const referenceTrajectory = cell.reference_trajectory;
+		const repetitions = cell.repetitions.map((rep) => {
+			const base = {
+				repetitionIndex: rep.repetitionIndex,
+				durationMs: rep.durationMs,
+				assertionResults: rep.assertionResults
+			};
+			if (rep.error) {
+				base.error = {
+					message: rep.error.message,
+					diagnostics: rep.error.diagnostics
+				};
+				return base;
+			}
+			if (rep.adapterResult) {
+				base.trajectory = {
+					...rep.adapterResult.view,
+					schemaVersion: "1.0"
+				};
+				base.diagnostics = rep.adapterResult.diagnostics;
+				const artifacts = {};
+				if (includeTranscript) artifacts.transcript = trajectoryToTranscript(rep.adapterResult.view, prompt);
+				if (includeRaw) {
+					const raw = extractRawEvents(rep.adapterResult);
+					if (raw) artifacts.rawStreamEvents = raw;
+				}
+				if (Object.keys(artifacts).length > 0) base.artifacts = artifacts;
+			}
+			const graded = options.grading?.results.find((r) => r.caseId === cell.caseId && r.cellLabel === cell.cell.label && r.repetitionIndex === rep.repetitionIndex);
+			if (graded) base.outcomeGrades = {
+				judge,
+				expectations: graded.expectations,
+				summary: graded.summary,
+				evalFeedback: graded.evalFeedback,
+				error: graded.graderError
+			};
+			return enrichRepetitionWithInterchange(base, referenceTrajectory);
+		});
+		return {
+			caseId: cell.caseId,
+			category: cell.category,
+			notes: cell.notes,
+			prompt: cell.prompt,
+			expectations: cell.expectations,
+			reference_trajectory: cell.reference_trajectory,
+			human_ratings: cell.human_ratings,
+			cellLabel: cell.cell.label,
+			axes: cell.cell.axes,
+			assertionStats: cell.assertionStats,
+			adapterErrors: cell.adapterErrors,
+			behavioralPass: cell.passed,
+			outcomePass: outcomePassForCell(cell.caseId, cell.cell.label, repetitions),
+			repetitions
+		};
+	});
+	const cellsPassed = cells.filter((c) => c.behavioralPass).length;
+	const gradedCells = cells.filter((c) => c.outcomePass !== void 0);
+	const outcomePass = gradedCells.length > 0 ? gradedCells.every((c) => c.outcomePass === true) : void 0;
+	return {
+		schemaVersion: "1.0",
+		runId: options.runId ?? randomUUID(),
+		startedAt: report.startedAt,
+		durationMs: report.durationMs,
+		suite: options.suite,
+		harness: {
+			adapter: options.harness?.adapter ?? "claude-code",
+			frameworkVersion: options.harness?.frameworkVersion,
+			harnessVersion: options.harness?.harnessVersion
+		},
+		provenance: options.provenance,
+		summary: {
+			cellsTotal: cells.length,
+			cellsPassed,
+			behavioralPass: cellsPassed === cells.length,
+			outcomePass
+		},
+		cells
+	};
+}
+/** Build envelope from on-disk report + optional grading JSON paths. */
+async function buildEvalRunEnvelopeFromFiles(reportPath, options = {}) {
+	const reportText = await readFile(reportPath, "utf8");
+	const report = JSON.parse(reportText);
+	let grading = options.grading;
+	if (options.gradingPath) {
+		const gradingText = await readFile(options.gradingPath, "utf8");
+		const parsed = JSON.parse(gradingText);
+		grading = {
+			gradedAt: parsed.gradedAt,
+			sourceReport: parsed.sourceReport,
+			results: parsed.results,
+			judge: options.grading?.judge ?? { id: "harness-eval/claude-grader" }
+		};
+	}
+	let suite = options.suite;
+	if (options.suitePath) {
+		const content = await readFile(options.suitePath, "utf8");
+		suite = {
+			...suite,
+			uri: options.suitePath,
+			contentHash: createHash("sha256").update(content).digest("hex")
+		};
+	}
+	return buildEvalRunEnvelope(report, {
+		...options,
+		suite,
+		grading
+	});
+}
+//#endregion
+export { TRAJECTORY_SCHEMA_VERSION as A, gradeReport as C, emitOtel as D, createClaudeGrader as E, trajectoryToOtlp as O, resolveGradeOptions as S, trajectoryToTranscript as T, trajectoryRecall as _, toProtoInstances as a, formatGradingConsole as b, toolCallValid as c, toolParameterKvMatch as d, computeTrajectoryMetrics as f, trajectoryPrecision as g, trajectoryInOrderMatch as h, toAgentTrace as i, EVAL_RUN_SCHEMA_VERSION as k, toolNameMatch as l, trajectoryExactMatch as m, buildEvalRunEnvelopeFromFiles as n, toTrajectory as o, trajectoryAnyOrderMatch as p, enrichRepetitionWithInterchange as r, computeToolCallMetrics as s, buildEvalRunEnvelope as t, toolParameterKeyMatch as u, trajectorySingleToolUse as v, loadSuiteReport as w, gradingReportPassed as x, formatReport as y };
+
+//# sourceMappingURL=build-DsVJ_UeU.js.map