@alis-build/harness-eval 0.1.2 → 0.1.4

package/dist/{claude-code-DZ4Vkgp6.js → claude-code-C_7hxC8z.js}

@@ -1,235 +1,9 @@
 import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.js";
+import { n as TrajectoryBuilder, t as AdapterError } from "./types-Bac8_Ixb.js";
 import { spawn } from "node:child_process";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-//#region src/types/stream.ts
-/** Type guards. Prefer these over manual `e.type === "..."` checks at call sites. */
-function isSystemInit(e) {
-	return e.type === "system" && e.subtype === "init";
-}
-function isSystemRetry(e) {
-	return e.type === "system" && e.subtype === "api_retry";
-}
-function isAssistantMessage(e) {
-	return e.type === "assistant";
-}
-function isUserMessage(e) {
-	return e.type === "user";
-}
-function isResult(e) {
-	return e.type === "result";
-}
-function isTextBlock(b) {
-	return b.type === "text";
-}
-function isToolUseBlock(b) {
-	return b.type === "tool_use";
-}
-function isToolResultBlock(b) {
-	return b.type === "tool_result";
-}
-//#endregion
-//#region src/types/trajectory.ts
-/**
-* Extract the MCP namespace prefix from a tool name.
-*
-* Claude Code formats MCP tool names as `mcp__<server>__<tool>`. The namespace
-* is the first two segments joined: `mcp__<server>`. Returns null for non-MCP
-* tool names (built-ins like `Bash`, `Read`, `Edit`).
-*
-* @example
-*   namespaceOf("mcp__api__search_skills") // "mcp__api"
-*   namespaceOf("Bash")                     // null
-*/
-function namespaceOf(toolName) {
-	if (!toolName.startsWith("mcp__")) return null;
-	const parts = toolName.split("__");
-	if (parts.length < 3) return null;
-	return `${parts[0]}__${parts[1]}`;
-}
-//#endregion
-//#region src/trajectory/builder.ts
-/**
-* TrajectoryBuilder — consumes a stream of {@link StreamEvent} values and
-* produces a {@link TrajectoryView}.
-*
-* State machine: the builder is a small, tolerant state machine. Invariants:
-*
-*   - Exactly one `system/init` event opens the session. The builder requires
-*     it to be present before `build()`.
-*   - Each `assistant` event begins a new turn. Text blocks accumulate into
-*     the turn's text; `tool_use` blocks become `ToolCall` records.
-*   - `user` events with `tool_result` blocks deliver tool results back. We
-*     match them to pending calls by `tool_use_id`.
-*   - One `result` event closes the session and carries aggregate usage.
-*
-* The builder is *tolerant of partial streams*: a process killed mid-run
-* produces a coherent (but flagged) view. Tool calls without matching results
-* keep `result: null`. The `success` flag reflects whether a successful result
-* event was actually observed.
-*
-* Why a class (not a reducer)?
-*   The internal `pendingCalls` map is mutable by design — we modify ToolCall
-*   objects in place when results arrive, so other parts of the view (which
-*   hold references to the same objects) see the update for free. A reducer
-*   would force a deep copy per result event, which is wasteful and would
-*   complicate identity-based queries.
-*/
-var TrajectoryBuilder = class {
-	meta = null;
-	sessionStartTs = null;
-	turns = [];
-	allToolCalls = [];
-	/**
-	* tool_use_id → ToolCall, for matching results back to calls.
-	* Entries are removed once a result is observed.
-	*/
-	pendingCalls = /* @__PURE__ */ new Map();
-	retries = [];
-	finalUsage = null;
-	finalCostUsd = 0;
-	finalDurationMs = 0;
-	finalNumTurns = 0;
-	finalResultText = "";
-	sawResultEvent = false;
-	resultIsError = false;
-	/**
-	* Consume one event. Safe to call with events in stream order.
-	*
-	* Unknown event types are silently ignored — the schema evolves and we
-	* don't want CI to break on a new event type we haven't modelled.
-	*/
-	consume(event) {
-		if (isSystemInit(event)) {
-			this.meta = {
-				sessionId: event.session_id,
-				model: event.model,
-				cwd: event.cwd,
-				permissionMode: event.permissionMode,
-				availableTools: event.tools ?? [],
-				mcpServers: (event.mcp_servers ?? []).map((s) => ({
-					name: s.name,
-					status: s.status
-				}))
-			};
-			this.sessionStartTs = Date.now();
-			return;
-		}
-		if (event.type === "system" && event.subtype === "api_retry") {
-			this.retries.push({
-				offsetMs: this.sessionStartTs ? Date.now() - this.sessionStartTs : 0,
-				raw: event
-			});
-			return;
-		}
-		if (isAssistantMessage(event)) {
-			this.handleAssistantMessage(event);
-			return;
-		}
-		if (isUserMessage(event)) {
-			this.handleUserMessage(event);
-			return;
-		}
-		if (isResult(event)) {
-			this.sawResultEvent = true;
-			this.resultIsError = event.is_error;
-			this.finalUsage = event.usage ?? null;
-			this.finalCostUsd = event.total_cost_usd ?? 0;
-			this.finalDurationMs = event.duration_ms ?? 0;
-			this.finalNumTurns = event.num_turns ?? 0;
-			this.finalResultText = event.result ?? "";
-			return;
-		}
-	}
-	/**
-	* Finalize the view. Call after consuming the last event from the stream.
-	*
-	* Throws if no `system/init` was observed — at that point we have no model,
-	* no session id, and no available-tools list, which means assertions like
-	* "called any mcp__api__* tool" can't even be evaluated meaningfully.
-	*/
-	build() {
-		if (this.meta === null) throw new Error("TrajectoryBuilder.build() called before any system/init event was observed. The harness may have failed to start, or the stream was truncated before init.");
-		const lastTurn = this.turns[this.turns.length - 1];
-		const accumulatedText = this.turns.map((t) => t.text).filter((t) => t.length > 0).join("\n\n").trim();
-		return {
-			meta: this.meta,
-			toolCalls: this.allToolCalls,
-			turns: this.turns,
-			finalResponse: accumulatedText || this.finalResultText,
-			finalStopReason: lastTurn?.stopReason ?? null,
-			usage: {
-				inputTokens: this.finalUsage?.input_tokens ?? 0,
-				outputTokens: this.finalUsage?.output_tokens ?? 0,
-				totalCostUsd: this.finalCostUsd,
-				durationMs: this.finalDurationMs,
-				numTurns: this.finalNumTurns || this.turns.length
-			},
-			retries: this.retries,
-			success: this.sawResultEvent && !this.resultIsError
-		};
-	}
-	handleAssistantMessage(event) {
-		const turnIndex = this.turns.length;
-		const textChunks = [];
-		const toolCallsThisTurn = [];
-		for (const block of event.message.content) {
-			if (isTextBlock(block)) {
-				textChunks.push(block.text);
-				continue;
-			}
-			if (isToolUseBlock(block)) {
-				const call = {
-					name: block.name,
-					namespace: namespaceOf(block.name),
-					callId: block.id,
-					args: block.input,
-					result: null,
-					isError: false,
-					turnIndex,
-					callIndex: this.allToolCalls.length
-				};
-				this.allToolCalls.push(call);
-				this.pendingCalls.set(block.id, call);
-				toolCallsThisTurn.push(call);
-				continue;
-			}
-		}
-		this.turns.push({
-			turnIndex,
-			text: textChunks.join("").trim(),
-			toolCalls: toolCallsThisTurn,
-			stopReason: event.message.stop_reason ?? null
-		});
-	}
-	handleUserMessage(event) {
-		const content = event.message.content;
-		if (typeof content === "string") return;
-		for (const block of content) {
-			if (!isToolResultBlock(block)) continue;
-			const call = this.pendingCalls.get(block.tool_use_id);
-			if (!call) continue;
-			call.result = block.content;
-			call.isError = block.is_error ?? false;
-			this.pendingCalls.delete(block.tool_use_id);
-		}
-	}
-};
-/**
-* Convenience: drain an async iterable of events through a fresh builder.
-*
-* Suitable when you have the full event stream and just want the view.
-* For interactive/incremental scenarios (e.g. surfacing partial state in a UI)
-* instantiate {@link TrajectoryBuilder} directly and call `consume()` /
-* `build()` yourself.
-*/
-async function buildTrajectory(events) {
-	const builder = new TrajectoryBuilder();
-	for await (const event of events) builder.consume(event);
-	return builder.build();
-}
-//#endregion
 //#region src/parsers/stream-json.ts
 /**
 * Parse a readable stream of NDJSON into a sequence of typed stream-json events.
@@ -281,22 +55,6 @@ function tryParseLine(line) {
 	}
 }
 //#endregion
-//#region src/adapters/types.ts
-/**
-* Thrown when the harness fails to produce a usable trajectory.
-*
-* Most commonly this means the process failed before emitting a usable
-* session init event. Inspect `diagnostics.stderr` for the cause.
-*/
-var AdapterError = class extends Error {
-	diagnostics;
-	constructor(message, diagnostics) {
-		super(message);
-		this.diagnostics = diagnostics;
-		this.name = "AdapterError";
-	}
-};
-//#endregion
 //#region src/adapters/claude-code/flags.ts
 /** Append repeated `--flag value` pairs for array config fields. */
 function pushRepeatableFlag(args, flag, values) {
@@ -587,6 +345,6 @@ const claudeCodeAdapter = {
 	run: runClaudeCode
 };
 //#endregion
-export { isUserMessage as _, AdapterError as a, buildTrajectory as c, isResult as d, isSystemInit as f, isToolUseBlock as g, isToolResultBlock as h, buildJudgeArgs as i, namespaceOf as l, isTextBlock as m, claude_code_exports as n, parseStreamJson as o, isSystemRetry as p, runClaudeCode as r, TrajectoryBuilder as s, claudeCodeAdapter as t, isAssistantMessage as u };
+export { parseStreamJson as a, buildJudgeArgs as i, claude_code_exports as n, runClaudeCode as r, claudeCodeAdapter as t };
 
-//# sourceMappingURL=claude-code-DZ4Vkgp6.js.map
+//# sourceMappingURL=claude-code-C_7hxC8z.js.map

package/dist/claude-code-C_7hxC8z.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-code-C_7hxC8z.js","names":[],"sources":["../src/parsers/stream-json.ts","../src/adapters/claude-code/flags.ts","../src/adapters/claude-code/process.ts","../src/adapters/claude-code/index.ts"],"sourcesContent":["/**\n * Line-buffered NDJSON parser for Claude Code's `--output-format stream-json`.\n *\n * Claude Code emits one JSON object per line on stdout. The parser:\n *   - buffers across chunk boundaries (a single JSON line may arrive in two reads)\n *   - skips empty lines (defensive — shouldn't occur, but harmless if it does)\n *   - emits a discriminated `ParseResult` per line so callers can decide whether\n *     a malformed line should abort the run or just be logged.\n *\n * Why a generator (and not a Transform stream)?\n *   The eval adapter consumes events sequentially and synchronously updates a\n *   builder. Async iteration is the simplest interface for that pattern and\n *   composes cleanly with `for await` in the adapter. A Transform would force\n *   the builder into event-handler style.\n */\n\nimport type { Readable } from \"node:stream\";\nimport type { StreamEvent } from \"../types/stream\";\n\n/**\n * Result of attempting to parse a single line.\n *\n * Successful parses yield `{ ok: true }` with the typed event and the raw line\n * (kept for diagnostics and OTel `events.attributes.raw`). Failed parses yield\n * `{ ok: false }` with the parse error and the raw line — callers can log,\n * skip, or fail the run as they see fit.\n */\nexport type ParseResult =\n  | { ok: true; event: StreamEvent; rawLine: string }\n  | { ok: false; error: Error; rawLine: string };\n\n/**\n * Parse a readable stream of NDJSON into a sequence of typed stream-json events.\n *\n * @example\n *   const child = spawn(\"claude\", [\"-p\", prompt, \"--output-format\", \"stream-json\", \"--verbose\"]);\n *   for await (const result of parseStreamJson(child.stdout)) {\n *     if (result.ok) builder.consume(result.event);\n *     else console.warn(\"malformed stream line:\", result.rawLine, result.error);\n *   }\n */\nexport async function* parseStreamJson(\n  stream: Readable,\n): AsyncGenerator<ParseResult, void, void> {\n  let buffer = \"\";\n  // The Node child_process stdout is a binary stream by default. Setting the\n  // encoding here means `for await (const chunk of stream)` yields strings.\n  stream.setEncoding(\"utf8\");\n\n  for await (const chunk of stream) {\n    buffer += chunk as string;\n\n    // Drain every complete line currently in the buffer before reading more.\n    // Multiple JSON objects can arrive in one chunk (e.g. when the harness\n    // emits a burst of events at session start).\n    let newlineIdx: number;\n    while ((newlineIdx = buffer.indexOf(\"\\n\")) !== -1) {\n      const line = buffer.slice(0, newlineIdx).trim();\n      buffer = buffer.slice(newlineIdx + 1);\n      if (line.length === 0) continue;\n      yield tryParseLine(line);\n    }\n  }\n\n  // Flush any trailing content that arrived without a final newline. Stream-json\n  // typically ends with a newline-terminated `result` event, but a killed\n  // process may not flush, so we still try to emit what we have.\n  const trailing = buffer.trim();\n  if (trailing.length > 0) {\n    yield tryParseLine(trailing);\n  }\n}\n\n/**\n * Parse a single line. Extracted as a helper so the generator stays readable.\n *\n * Note: we do not validate the event structure beyond `JSON.parse`. Runtime\n * validation (e.g. zod) is overkill here — the schema is stable enough at\n * runtime, and the TrajectoryBuilder is tolerant of missing fields. Adding\n * validation would be premature.\n */\nfunction tryParseLine(line: string): ParseResult {\n  try {\n    const event = JSON.parse(line) as StreamEvent;\n    return { ok: true, event, rawLine: line };\n  } catch (err) {\n    return {\n      ok: false,\n      error: err instanceof Error ? err : new Error(String(err)),\n      rawLine: line,\n    };\n  }\n}\n","/**\n * Build CLI args for Claude Code judge subprocesses (JSON output, not stream-json).\n *\n * Shared flag assembly for harness runs (`buildArgs`) and LLM grading judges\n * (`buildJudgeArgs`).\n */\n\nimport type { ClaudeCodeAdapterConfig, ClaudeCodeOptions } from \"./types\";\n\n/** Append repeated `--flag value` pairs for array config fields. */\nfunction pushRepeatableFlag(args: string[], flag: string, values?: string[]): void {\n  if (!values) return;\n  for (const value of values) {\n    args.push(flag, value);\n  }\n}\n\n/**\n * Append an optional CLI flag. Boolean `true` emits the flag alone; other\n * scalars emit `--flag value`.\n */\nfunction pushOptionalFlag(\n  args: string[],\n  flag: string,\n  value: string | number | boolean | undefined,\n): void {\n  if (value === undefined) return;\n  if (typeof value === \"boolean\") {\n    if (value) args.push(flag);\n    return;\n  }\n  args.push(flag, String(value));\n}\n\n/** Append Claude Code CLI flags shared by harness runs and grading judges. */\nexport function appendClaudeCodeFlags(\n  args: string[],\n  config: ClaudeCodeOptions & { model?: string },\n): void {\n  pushRepeatableFlag(args, \"--plugin-dir\", config.pluginDirs);\n  pushRepeatableFlag(args, \"--plugin-url\", config.pluginUrls);\n  pushRepeatableFlag(args, \"--add-dir\", config.addDirs);\n\n  pushOptionalFlag(args, \"--mcp-config\", config.mcpConfig);\n  pushOptionalFlag(args, \"--model\", config.model);\n  pushOptionalFlag(args, \"--permission-mode\", config.permissionMode);\n  pushOptionalFlag(args, \"--effort\", config.effort);\n  pushOptionalFlag(args, \"--agent\", config.agent);\n  pushOptionalFlag(args, \"--fallback-model\", config.fallbackModel);\n  pushOptionalFlag(args, \"--tools\", config.tools);\n  pushOptionalFlag(args, \"--settings\", config.settings);\n  pushOptionalFlag(args, \"--setting-sources\", config.settingSources);\n  pushOptionalFlag(args, \"--max-turns\", config.maxTurns);\n  pushOptionalFlag(args, \"--max-budget-usd\", config.maxBudgetUsd);\n  pushOptionalFlag(args, \"--system-prompt\", config.systemPrompt);\n  pushOptionalFlag(args, \"--system-prompt-file\", config.systemPromptFile);\n  pushOptionalFlag(args, \"--append-system-prompt\", config.appendSystemPrompt);\n  pushOptionalFlag(\n    args,\n    \"--append-system-prompt-file\",\n    config.appendSystemPromptFile,\n  );\n  pushOptionalFlag(args, \"--debug\", config.debug);\n  pushOptionalFlag(args, \"--debug-file\", config.debugFile);\n\n  if (config.allowedTools && config.allowedTools.length > 0) {\n    args.push(\"--allowedTools\", config.allowedTools.join(\",\"));\n  }\n\n  if (config.disallowedTools && config.disallowedTools.length > 0) {\n    args.push(\"--disallowedTools\", config.disallowedTools.join(\",\"));\n  }\n\n  pushOptionalFlag(args, \"--strict-mcp-config\", config.strictMcpConfig);\n  pushOptionalFlag(args, \"--include-hook-events\", config.includeHookEvents);\n  pushOptionalFlag(args, \"--no-session-persistence\", config.noSessionPersistence);\n  pushOptionalFlag(args, \"--disable-slash-commands\", config.disableSlashCommands);\n  pushOptionalFlag(args, \"--bare\", config.bare);\n  pushOptionalFlag(args, \"--safe-mode\", config.safeMode);\n  pushOptionalFlag(\n    args,\n    \"--allow-dangerously-skip-permissions\",\n    config.allowDangerouslySkipPermissions,\n  );\n  pushOptionalFlag(\n    args,\n    \"--dangerously-skip-permissions\",\n    config.dangerouslySkipPermissions,\n  );\n}\n\n/**\n * Build the argument vector for spawning `claude`.\n *\n * Order matters only for flags that take values — value flags must come\n * after their flag name. Everything else is order-independent.\n */\nexport function buildArgs(config: ClaudeCodeAdapterConfig): string[] {\n  const args: string[] = [\n    \"-p\",\n    config.prompt,\n    \"--output-format\",\n    \"stream-json\",\n    \"--verbose\",\n  ];\n\n  appendClaudeCodeFlags(args, config);\n\n  return args;\n}\n\n/**\n * Build args for an LLM judge subprocess (`--output-format json`).\n *\n * Defaults permission mode to `bypassPermissions` so the judge does not\n * block on tool permission prompts during single-shot JSON grading.\n */\nexport function buildJudgeArgs(\n  prompt: string,\n  config: ClaudeCodeOptions & { model?: string } = {},\n): string[] {\n  const args: string[] = [\"-p\", prompt, \"--output-format\", \"json\"];\n  const permissionMode = config.permissionMode ?? \"bypassPermissions\";\n  appendClaudeCodeFlags(args, {\n    ...config,\n    permissionMode,\n  });\n  return args;\n}\n","/**\n * Process management for the Claude Code adapter.\n *\n * This module owns spawning, timeout, abort signal handling, and process-tree\n * teardown. The orchestrator (`index.ts`) consumes the returned handle —\n * reading stdout and waiting for completion — but doesn't worry about how\n * the process gets killed or how its config gets isolated.\n *\n * Why a separate module? Process management is the one part of the adapter\n * with real I/O complexity (process groups, signal escalation, temp-dir\n * lifecycle, env merging). Isolating it makes the orchestrator easy to read\n * and lets us swap the spawning logic if we later need to, e.g., wrap claude\n * in a sandbox runner.\n */\n\nimport { spawn, type ChildProcess } from \"node:child_process\";\nimport { mkdtemp, rm } from \"node:fs/promises\";\nimport { tmpdir } from \"node:os\";\nimport { join } from \"node:path\";\nimport type { Readable } from \"node:stream\";\n\nimport { buildArgs } from \"./flags\";\nimport type { ClaudeCodeAdapterConfig } from \"./types\";\n\n/** Default hard timeout per run. Tunable via config.timeoutMs. */\nconst DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;\n\n/**\n * Grace period between SIGTERM and SIGKILL. Most processes shut down cleanly\n * within a few seconds; this gives them that chance while preventing CI from\n * hanging indefinitely on a stuck child.\n */\nconst KILL_GRACE_MS = 5_000;\n\n/**\n * Handle to a spawned `claude` process. The orchestrator drives it:\n *   - Read `stdout` (typically via parseStreamJson).\n *   - Await `done` to learn the exit state.\n *   - Await `stderrCollected` for diagnostic stderr.\n *   - Check `timedOut()` after exit to distinguish kill-by-timeout from\n *     normal termination.\n *   - Call `cleanup()` after all of the above to remove the temp config dir.\n */\nexport interface SpawnedClaude {\n  stdout: Readable;\n  done: Promise<{ exitCode: number | null; signal: NodeJS.Signals | null }>;\n  stderrCollected: Promise<string>;\n  timedOut: () => boolean;\n  cleanup: () => Promise<void>;\n}\n\n/**\n * Spawn `claude` in headless mode with isolated config and a process-group\n * lifecycle. See {@link SpawnedClaude} for how to consume the result.\n *\n * **Kill sequence:** timeout and abort both follow the same two-step path:\n * `SIGTERM` to the process group, then `SIGKILL` after {@link KILL_GRACE_MS}\n * if the group is still alive. This avoids leaving MCP/tool subprocesses\n * running while still giving claude a chance to flush stream-json output.\n *\n * @param config - Adapter options; `timeoutMs`, `signal`, and `isolateConfig`\n *   control lifecycle and config isolation.\n */\nexport async function spawnClaude(\n  config: ClaudeCodeAdapterConfig,\n): Promise<SpawnedClaude> {\n  const binary = config.binary ?? \"claude\";\n  const args = buildArgs(config);\n\n  const isolateConfig = config.isolateConfig !== false;\n\n  // Isolated runs use a fresh temp dir so plugins/settings don't leak between\n  // reps. Non-isolated runs inherit the caller's Claude login and plugins.\n  const tempConfigDir = isolateConfig\n    ? await mkdtemp(join(tmpdir(), \"harness-eval-\"))\n    : null;\n\n  const env: Record<string, string | undefined> = {\n    ...process.env,\n    ...config.env,\n  };\n  if (tempConfigDir) {\n    // Override after ...env so callers can't accidentally un-isolate.\n    env.CLAUDE_CONFIG_DIR = tempConfigDir;\n  }\n\n  const child = spawn(binary, args, {\n    cwd: config.cwd ?? process.cwd(),\n    env,\n    stdio: [\"ignore\", \"pipe\", \"pipe\"],\n    // detached: true means the child becomes the leader of its own process\n    // group. We exploit this to kill the entire group (including any MCP\n    // server subprocesses and tool processes) on timeout/abort.\n    detached: true,\n  });\n\n\n  // `timedOut` is set only by the hard timeout timer, not by abort — callers\n  // use it to distinguish \"ran too long\" from user cancellation or normal exit.\n  let timedOut = false;\n  let killEscalation: NodeJS.Timeout | null = null;\n  const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n\n  /**\n   * Arm (or re-arm) the SIGKILL fallback. Each SIGTERM attempt gets its own\n   * grace window so a slow shutdown doesn't leave orphaned MCP servers.\n   */\n  const scheduleKillEscalation = () => {\n    if (killEscalation) clearTimeout(killEscalation);\n    killEscalation = setTimeout(\n      () => killTree(child, \"SIGKILL\"),\n      KILL_GRACE_MS,\n    );\n  };\n\n  const timeoutTimer = setTimeout(() => {\n    timedOut = true;\n    killTree(child, \"SIGTERM\");\n    scheduleKillEscalation();\n  }, timeoutMs);\n\n  // AbortSignal cancellation mirrors timeout kills but does not flip `timedOut`.\n  const onAbort = () => {\n    killTree(child, \"SIGTERM\");\n    scheduleKillEscalation();\n  };\n  config.signal?.addEventListener(\"abort\", onAbort, { once: true });\n\n\n  // Drain stderr eagerly so the OS-level buffer never fills and stalls the\n  // child (Node child processes will block on a full pipe).\n  const stderrChunks: string[] = [];\n  child.stderr?.setEncoding(\"utf8\");\n  child.stderr?.on(\"data\", (chunk: string) => {\n    stderrChunks.push(chunk);\n  });\n\n  const stderrCollected = new Promise<string>((resolve) => {\n    const finalize = () => resolve(stderrChunks.join(\"\"));\n    child.stderr?.on(\"end\", finalize);\n    // Errors during stderr capture shouldn't fail the whole run; we just\n    // return what we've buffered so far.\n    child.stderr?.on(\"error\", finalize);\n  });\n\n\n  // Resolve once the process exits or fails to spawn. Guard against double\n  // settlement because both `close` and `error` can fire in edge cases.\n  const done = new Promise<{\n    exitCode: number | null;\n    signal: NodeJS.Signals | null;\n  }>((resolve) => {\n    let settled = false;\n    const finalize = (\n      exitCode: number | null,\n      signal: NodeJS.Signals | null,\n    ) => {\n      if (settled) return;\n      settled = true;\n      // Tear down timers/listeners so a late timeout cannot SIGKILL a reused PID.\n      clearTimeout(timeoutTimer);\n      if (killEscalation) clearTimeout(killEscalation);\n      config.signal?.removeEventListener(\"abort\", onAbort);\n      resolve({ exitCode, signal });\n    };\n\n    child.on(\"close\", (code, signal) => finalize(code, signal));\n    // ENOENT and other spawn failures emit `error` — `close` may not follow.\n    child.on(\"error\", () => finalize(null, null));\n  });\n\n\n  const cleanup = async () => {\n    if (!tempConfigDir) return;\n    try {\n      await rm(tempConfigDir, { recursive: true, force: true });\n    } catch {\n      // Best-effort. A leftover temp dir is annoying but not catastrophic;\n      // we don't want to fail the run for it.\n    }\n  };\n\n  // stdout is guaranteed non-null because we passed `stdio: [..., \"pipe\", ...]`.\n  // The `!` is safe; the alternative would be a redundant runtime check that\n  // could never fire.\n  return {\n    stdout: child.stdout!,\n    done,\n    stderrCollected,\n    timedOut: () => timedOut,\n    cleanup,\n  };\n}\n\n/**\n * Kill the child's process group, then fall back to the bare PID if the\n * group is already gone. This catches MCP server subprocesses and tool\n * processes spawned by claude.\n *\n * **Signal escalation:** callers typically invoke this first with `SIGTERM`,\n * then again with `SIGKILL` after {@link KILL_GRACE_MS}. The group kill is\n * essential — a bare `child.kill()` would leave MCP servers running.\n *\n * **Platform edge case:** when the group leader exits first, `kill(-pid)`\n * throws `ESRCH`. The single-PID fallback covers that without failing the\n * adapter run.\n *\n * @param child - Spawned process handle from {@link spawn}.\n * @param signal - POSIX signal to deliver (`SIGTERM` or `SIGKILL` in practice).\n */\nfunction killTree(child: ChildProcess, signal: NodeJS.Signals): void {\n  if (child.pid === undefined) return;\n  try {\n    // Negative PID targets the entire process group (requires detached spawn).\n    process.kill(-child.pid, signal);\n  } catch {\n    try {\n      // Group already reaped — try the leader PID directly.\n      child.kill(signal);\n    } catch {\n      // Process fully gone; nothing to do.\n    }\n  }\n}\n","/**\n * Claude Code adapter — public API.\n */\n\nimport { parseStreamJson } from \"../../parsers/stream-json\";\nimport { TrajectoryBuilder } from \"../../trajectory/builder\";\nimport type { StreamEvent } from \"../../types/stream\";\n\nimport { AdapterError } from \"../types\";\nimport { spawnClaude } from \"./process\";\nimport type {\n  AdapterDiagnostics,\n  ClaudeCodeAdapterConfig,\n  ClaudeCodeAdapterResult,\n  ParseErrorRecord,\n} from \"./types\";\nimport type { HarnessAdapter } from \"../types\";\n\nexport { AdapterError } from \"../types\";\nexport type {\n  AdapterDiagnostics,\n  AdapterResult,\n  ClaudeCodeAdapterConfig,\n  ClaudeCodeAdapterResult,\n  ClaudeCodeOptions,\n  ParseErrorRecord,\n  PermissionMode,\n} from \"./types\";\n\n/**\n * Run Claude Code in headless mode and return a trajectory.\n */\nexport async function runClaudeCode(\n  config: ClaudeCodeAdapterConfig,\n): Promise<ClaudeCodeAdapterResult> {\n  const startTs = Date.now();\n  const spawned = await spawnClaude(config);\n\n  const builder = new TrajectoryBuilder();\n  const rawEvents: StreamEvent[] = [];\n  const parseErrors: ParseErrorRecord[] = [];\n\n  try {\n    for await (const result of parseStreamJson(spawned.stdout)) {\n      if (result.ok) {\n        builder.consume(result.event);\n        rawEvents.push(result.event);\n      } else {\n        parseErrors.push({\n          line: result.rawLine,\n          error: result.error.message,\n        });\n      }\n    }\n\n    const [{ exitCode, signal }, stderr] = await Promise.all([\n      spawned.done,\n      spawned.stderrCollected,\n    ]);\n\n    const diagnostics: AdapterDiagnostics = {\n      exitCode,\n      signal,\n      stderr,\n      parseErrors,\n      timedOut: spawned.timedOut(),\n      durationMs: Date.now() - startTs,\n    };\n\n    let view;\n    try {\n      view = builder.build();\n    } catch (err) {\n      const message = err instanceof Error ? err.message : String(err);\n      throw new AdapterError(\n        `harness produced no usable trajectory: ${message}`,\n        diagnostics,\n      );\n    }\n\n    return { view, diagnostics, rawEvents };\n  } finally {\n    await spawned.cleanup();\n  }\n}\n\n/** Registered {@link HarnessAdapter} for Claude Code headless runs. */\nexport const claudeCodeAdapter: HarnessAdapter<ClaudeCodeAdapterConfig> = {\n  id: \"claude-code\",\n  run: runClaudeCode,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;AAyCA,gBAAuB,gBACrB,QACyC;CACzC,IAAI,SAAS;CAGb,OAAO,YAAY,MAAM;CAEzB,WAAW,MAAM,SAAS,QAAQ;EAChC,UAAU;EAKV,IAAI;EACJ,QAAQ,aAAa,OAAO,QAAQ,IAAI,OAAO,IAAI;GACjD,MAAM,OAAO,OAAO,MAAM,GAAG,UAAU,CAAC,CAAC,KAAK;GAC9C,SAAS,OAAO,MAAM,aAAa,CAAC;GACpC,IAAI,KAAK,WAAW,GAAG;GACvB,MAAM,aAAa,IAAI;EACzB;CACF;CAKA,MAAM,WAAW,OAAO,KAAK;CAC7B,IAAI,SAAS,SAAS,GACpB,MAAM,aAAa,QAAQ;AAE/B;;;;;;;;;AAUA,SAAS,aAAa,MAA2B;CAC/C,IAAI;EAEF,OAAO;GAAE,IAAI;GAAM,OADL,KAAK,MAAM,IACF;GAAG,SAAS;EAAK;CAC1C,SAAS,KAAK;EACZ,OAAO;GACL,IAAI;GACJ,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;GACzD,SAAS;EACX;CACF;AACF;;;;AClFA,SAAS,mBAAmB,MAAgB,MAAc,QAAyB;CACjF,IAAI,CAAC,QAAQ;CACb,KAAK,MAAM,SAAS,QAClB,KAAK,KAAK,MAAM,KAAK;AAEzB;;;;;AAMA,SAAS,iBACP,MACA,MACA,OACM;CACN,IAAI,UAAU,KAAA,GAAW;CACzB,IAAI,OAAO,UAAU,WAAW;EAC9B,IAAI,OAAO,KAAK,KAAK,IAAI;EACzB;CACF;CACA,KAAK,KAAK,MAAM,OAAO,KAAK,CAAC;AAC/B;;AAGA,SAAgB,sBACd,MACA,QACM;CACN,mBAAmB,MAAM,gBAAgB,OAAO,UAAU;CAC1D,mBAAmB,MAAM,gBAAgB,OAAO,UAAU;CAC1D,mBAAmB,MAAM,aAAa,OAAO,OAAO;CAEpD,iBAAiB,MAAM,gBAAgB,OAAO,SAAS;CACvD,iBAAiB,MAAM,WAAW,OAAO,KAAK;CAC9C,iBAAiB,MAAM,qBAAqB,OAAO,cAAc;CACjE,iBAAiB,MAAM,YAAY,OAAO,MAAM;CAChD,iBAAiB,MAAM,WAAW,OAAO,KAAK;CAC9C,iBAAiB,MAAM,oBAAoB,OAAO,aAAa;CAC/D,iBAAiB,MAAM,WAAW,OAAO,KAAK;CAC9C,iBAAiB,MAAM,cAAc,OAAO,QAAQ;CACpD,iBAAiB,MAAM,qBAAqB,OAAO,cAAc;CACjE,iBAAiB,MAAM,eAAe,OAAO,QAAQ;CACrD,iBAAiB,MAAM,oBAAoB,OAAO,YAAY;CAC9D,iBAAiB,MAAM,mBAAmB,OAAO,YAAY;CAC7D,iBAAiB,MAAM,wBAAwB,OAAO,gBAAgB;CACtE,iBAAiB,MAAM,0BAA0B,OAAO,kBAAkB;CAC1E,iBACE,MACA,+BACA,OAAO,sBACT;CACA,iBAAiB,MAAM,WAAW,OAAO,KAAK;CAC9C,iBAAiB,MAAM,gBAAgB,OAAO,SAAS;CAEvD,IAAI,OAAO,gBAAgB,OAAO,aAAa,SAAS,GACtD,KAAK,KAAK,kBAAkB,OAAO,aAAa,KAAK,GAAG,CAAC;CAG3D,IAAI,OAAO,mBAAmB,OAAO,gBAAgB,SAAS,GAC5D,KAAK,KAAK,qBAAqB,OAAO,gBAAgB,KAAK,GAAG,CAAC;CAGjE,iBAAiB,MAAM,uBAAuB,OAAO,eAAe;CACpE,iBAAiB,MAAM,yBAAyB,OAAO,iBAAiB;CACxE,iBAAiB,MAAM,4BAA4B,OAAO,oBAAoB;CAC9E,iBAAiB,MAAM,4BAA4B,OAAO,oBAAoB;CAC9E,iBAAiB,MAAM,UAAU,OAAO,IAAI;CAC5C,iBAAiB,MAAM,eAAe,OAAO,QAAQ;CACrD,iBACE,MACA,wCACA,OAAO,+BACT;CACA,iBACE,MACA,kCACA,OAAO,0BACT;AACF;;;;;;;AAQA,SAAgB,UAAU,QAA2C;CACnE,MAAM,OAAiB;EACrB;EACA,OAAO;EACP;EACA;EACA;CACF;CAEA,sBAAsB,MAAM,MAAM;CAElC,OAAO;AACT;;;;;;;AAQA,SAAgB,eACd,QACA,SAAiD,CAAC,GACxC;CACV,MAAM,OAAiB;EAAC;EAAM;EAAQ;EAAmB;CAAM;CAC/D,MAAM,iBAAiB,OAAO,kBAAkB;CAChD,sBAAsB,MAAM;EAC1B,GAAG;EACH;CACF,CAAC;CACD,OAAO;AACT;;;;;;;;;;;;;;;;;;ACvGA,MAAM,qBAAqB,MAAS;;;;;;AAOpC,MAAM,gBAAgB;;;;;;;;;;;;;AA+BtB,eAAsB,YACpB,QACwB;CACxB,MAAM,SAAS,OAAO,UAAU;CAChC,MAAM,OAAO,UAAU,MAAM;CAM7B,MAAM,gBAJgB,OAAO,kBAAkB,QAK3C,MAAM,QAAQ,KAAK,OAAO,GAAG,eAAe,CAAC,IAC7C;CAEJ,MAAM,MAA0C;EAC9C,GAAG,QAAQ;EACX,GAAG,OAAO;CACZ;CACA,IAAI,eAEF,IAAI,oBAAoB;CAG1B,MAAM,QAAQ,MAAM,QAAQ,MAAM;EAChC,KAAK,OAAO,OAAO,QAAQ,IAAI;EAC/B;EACA,OAAO;GAAC;GAAU;GAAQ;EAAM;EAIhC,UAAU;CACZ,CAAC;CAKD,IAAI,WAAW;CACf,IAAI,iBAAwC;CAC5C,MAAM,YAAY,OAAO,aAAa;;;;;CAMtC,MAAM,+BAA+B;EACnC,IAAI,gBAAgB,aAAa,cAAc;EAC/C,iBAAiB,iBACT,SAAS,OAAO,SAAS,GAC/B,aACF;CACF;CAEA,MAAM,eAAe,iBAAiB;EACpC,WAAW;EACX,SAAS,OAAO,SAAS;EACzB,uBAAuB;CACzB,GAAG,SAAS;CAGZ,MAAM,gBAAgB;EACpB,SAAS,OAAO,SAAS;EACzB,uBAAuB;CACzB;CACA,OAAO,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAKhE,MAAM,eAAyB,CAAC;CAChC,MAAM,QAAQ,YAAY,MAAM;CAChC,MAAM,QAAQ,GAAG,SAAS,UAAkB;EAC1C,aAAa,KAAK,KAAK;CACzB,CAAC;CAED,MAAM,kBAAkB,IAAI,SAAiB,YAAY;EACvD,MAAM,iBAAiB,QAAQ,aAAa,KAAK,EAAE,CAAC;EACpD,MAAM,QAAQ,GAAG,OAAO,QAAQ;EAGhC,MAAM,QAAQ,GAAG,SAAS,QAAQ;CACpC,CAAC;CAKD,MAAM,OAAO,IAAI,SAGb,YAAY;EACd,IAAI,UAAU;EACd,MAAM,YACJ,UACA,WACG;GACH,IAAI,SAAS;GACb,UAAU;GAEV,aAAa,YAAY;GACzB,IAAI,gBAAgB,aAAa,cAAc;GAC/C,OAAO,QAAQ,oBAAoB,SAAS,OAAO;GACnD,QAAQ;IAAE;IAAU;GAAO,CAAC;EAC9B;EAEA,MAAM,GAAG,UAAU,MAAM,WAAW,SAAS,MAAM,MAAM,CAAC;EAE1D,MAAM,GAAG,eAAe,SAAS,MAAM,IAAI,CAAC;CAC9C,CAAC;CAGD,MAAM,UAAU,YAAY;EAC1B,IAAI,CAAC,eAAe;EACpB,IAAI;GACF,MAAM,GAAG,eAAe;IAAE,WAAW;IAAM,OAAO;GAAK,CAAC;EAC1D,QAAQ,CAGR;CACF;CAKA,OAAO;EACL,QAAQ,MAAM;EACd;EACA;EACA,gBAAgB;EAChB;CACF;AACF;;;;;;;;;;;;;;;;;AAkBA,SAAS,SAAS,OAAqB,QAA8B;CACnE,IAAI,MAAM,QAAQ,KAAA,GAAW;CAC7B,IAAI;EAEF,QAAQ,KAAK,CAAC,MAAM,KAAK,MAAM;CACjC,QAAQ;EACN,IAAI;GAEF,MAAM,KAAK,MAAM;EACnB,QAAQ,CAER;CACF;AACF;;;;;;;;;;;;;;AC/LA,eAAsB,cACpB,QACkC;CAClC,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,UAAU,MAAM,YAAY,MAAM;CAExC,MAAM,UAAU,IAAI,kBAAkB;CACtC,MAAM,YAA2B,CAAC;CAClC,MAAM,cAAkC,CAAC;CAEzC,IAAI;EACF,WAAW,MAAM,UAAU,gBAAgB,QAAQ,MAAM,GACvD,IAAI,OAAO,IAAI;GACb,QAAQ,QAAQ,OAAO,KAAK;GAC5B,UAAU,KAAK,OAAO,KAAK;EAC7B,OACE,YAAY,KAAK;GACf,MAAM,OAAO;GACb,OAAO,OAAO,MAAM;EACtB,CAAC;EAIL,MAAM,CAAC,EAAE,UAAU,UAAU,UAAU,MAAM,QAAQ,IAAI,CACvD,QAAQ,MACR,QAAQ,eACV,CAAC;EAED,MAAM,cAAkC;GACtC;GACA;GACA;GACA;GACA,UAAU,QAAQ,SAAS;GAC3B,YAAY,KAAK,IAAI,IAAI;EAC3B;EAEA,IAAI;EACJ,IAAI;GACF,OAAO,QAAQ,MAAM;EACvB,SAAS,KAAK;GAEZ,MAAM,IAAI,aACR,0CAFc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,KAG7D,WACF;EACF;EAEA,OAAO;GAAE;GAAM;GAAa;EAAU;CACxC,UAAU;EACR,MAAM,QAAQ,QAAQ;CACxB;AACF;;AAGA,MAAa,oBAA6D;CACxE,IAAI;CACJ,KAAK;AACP"}

package/dist/cli/bin.js

@@ -1,152 +1,10 @@
 #!/usr/bin/env node
-import { C as loadSuiteReport, D as trajectoryToOtlp, S as gradeReport, b as gradingReportPassed, i as buildEvalRunEnvelopeFromFiles, n as toTrajectory, t as toInstancesJsonl, v as formatReport, x as resolveGradeOptions, y as formatGradingConsole } from "../projections-BcX7w-f6.js";
-import { t as runSuite, u as getAdapter } from "../suite-Dlzl-HI0.js";
-import { i as loadGradingConfig, t as loadSuite } from "../loader-DcI0KfRX.js";
+import { F as loadSuiteReport, M as gradingReportPassed, N as resolveGradeOptions, P as gradeReport, a as envelopeCommand, c as getOptionInt, i as runPipeline, j as formatGradingConsole, l as hasOption, o as parseEnvelopeProjection, p as suiteDirectoryFromPath, r as trajectoryToOtlp, s as getOption, t as formatReport, u as parseArgs } from "../reporter-BKCJZRYr.js";
+import { t as runSuite, u as getAdapter } from "../suite-C3-8EjUW.js";
+import { i as loadGradingConfig, o as loadSuiteDocument, t as loadSuite } from "../loader-CiBm4Kf6.js";
 import { mkdir, readFile, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
+import { dirname, isAbsolute, join } from "node:path";
 import { fileURLToPath } from "node:url";
-//#region src/cli/args.ts
-/** Parse process argv into command, positional args, and options. */
-function parseArgs(argv) {
-	const positional = [];
-	const options = {};
-	let command;
-	const args = [...argv];
-	if (args.length > 0 && !args[0].startsWith("-")) command = args.shift();
-	for (let i = 0; i < args.length; i++) {
-		const arg = args[i];
-		if (arg === "--") {
-			positional.push(...args.slice(i + 1));
-			break;
-		}
-		if (arg.startsWith("--")) {
-			const key = arg.slice(2);
-			const next = args[i + 1];
-			if (next && !next.startsWith("-")) {
-				options[key] = next;
-				i++;
-			} else options[key] = true;
-		} else if (arg.startsWith("-") && arg.length === 2) {
-			const key = arg.slice(1);
-			const next = args[i + 1];
-			if (next && !next.startsWith("-")) {
-				options[key] = next;
-				i++;
-			} else options[key] = true;
-		} else positional.push(arg);
-	}
-	return {
-		command,
-		positional,
-		options
-	};
-}
-/** Return a string option value, or undefined when absent or boolean. */
-function getOption(options, name) {
-	const v = options[name];
-	return typeof v === "string" ? v : void 0;
-}
-/** Parse an integer option with fallback when absent or non-numeric. */
-function getOptionInt(options, name, defaultValue) {
-	const v = getOption(options, name);
-	if (v === void 0) return defaultValue;
-	const n = Number.parseInt(v, 10);
-	if (!Number.isFinite(n)) return defaultValue;
-	return n;
-}
-/** True when a boolean flag is set or explicitly `"true"`. */
-function hasOption(options, name) {
-	const v = options[name];
-	return v === true || typeof v === "string" && v === "true";
-}
-//#endregion
-//#region src/cli/commands/envelope.ts
-/**
-* `harness-eval envelope` — build EvalRunEnvelope and interchange projections.
-*
-* Reads a suite run report (and optional grading JSON), builds a versioned
-* {@link EvalRunEnvelope}, and serializes one of three projections:
-*
-*   - `envelope` — full nested JSON document (default)
-*   - `trajectory` — JSONL of {@link EvalDatasetRow} per repetition
-*   - `instances` — JSONL of {@link InstancesJsonlRow} for Vertex batch upload
-*
-* Exit code 0 when behavioral pass, 1 when any cell failed assertions.
-*/
-const PROJECTIONS = /* @__PURE__ */ new Set([
-	"envelope",
-	"trajectory",
-	"instances"
-]);
-/**
-* Parse and validate `--projection` CLI flag.
-*
-* @returns `"envelope"` when omitted; `undefined` when value is invalid.
-*/
-function parseEnvelopeProjection(value) {
-	if (value === void 0) return "envelope";
-	if (PROJECTIONS.has(value)) return value;
-}
-/**
-* Serialize an envelope to stdout/file string for the chosen projection.
-*
-* Trajectory and instances projections emit NDJSON (one JSON object per line).
-*/
-function serializeEnvelopeProjection(envelope, projection) {
-	switch (projection) {
-		case "trajectory": return `${toTrajectory(envelope).map((row) => JSON.stringify(row)).join("\n")}\n`;
-		case "instances": return `${toInstancesJsonl(envelope).map((row) => JSON.stringify(row)).join("\n")}\n`;
-		default: return `${JSON.stringify(envelope, null, 2)}\n`;
-	}
-}
-/** Read harness-eval package version for envelope harness.frameworkVersion. */
-async function readFrameworkVersion() {
-	try {
-		const text = await readFile(join(dirname(fileURLToPath(import.meta.url)), "../../../package.json"), "utf8");
-		return JSON.parse(text).version;
-	} catch {
-		return;
-	}
-}
-/**
-* CLI entry point for the `envelope` subcommand.
-*
-* @returns Process exit code: 0 on behavioral pass, 1 on failure, 2 on usage/error.
-*/
-async function envelopeCommand(args) {
-	const reportPath = args.positional[0];
-	if (!reportPath) {
-		console.error("usage: harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]");
-		return 2;
-	}
-	const outputPath = getOption(args.options, "output");
-	const gradingPath = getOption(args.options, "grading");
-	const suitePath = getOption(args.options, "suite");
-	const projection = parseEnvelopeProjection(getOption(args.options, "projection"));
-	if (!projection) {
-		console.error("invalid --projection; expected envelope, trajectory, or instances");
-		return 2;
-	}
-	let envelope;
-	try {
-		const frameworkVersion = await readFrameworkVersion();
-		envelope = await buildEvalRunEnvelopeFromFiles(reportPath, {
-			gradingPath,
-			suitePath,
-			includeTranscript: !hasOption(args.options, "no-transcript"),
-			includeRawStreamEvents: hasOption(args.options, "include-raw-stream-events"),
-			harness: { frameworkVersion }
-		});
-	} catch (err) {
-		console.error(err instanceof Error ? err.message : String(err));
-		return 2;
-	}
-	const serialized = serializeEnvelopeProjection(envelope, projection);
-	if (outputPath) await writeFile(outputPath, serialized, "utf8");
-	else process.stdout.write(serialized);
-	return envelope.summary.behavioralPass ? 0 : 1;
-}
-//#endregion
 //#region src/cli/commands/format.ts
 /**
 * `harness-eval format` command.
@@ -507,10 +365,11 @@ function optionalOptionInt(options, name) {
 async function gradeCommand(args) {
 	const reportPath = args.positional[0];
 	if (!reportPath) {
-		console.error("usage: harness-eval grade <report.json> [--config grading.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N]");
+		console.error("usage: harness-eval grade <report.json> [--config grading.yaml] [--suite suite.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N]");
 		return 2;
 	}
 	const configPath = getOption(args.options, "config");
+	const suitePath = getOption(args.options, "suite");
 	const expectationsPath = getOption(args.options, "expectations");
 	const outputPath = getOption(args.options, "output");
 	const model = getOption(args.options, "model");
@@ -521,8 +380,13 @@ async function gradeCommand(args) {
 	const progressMode = resolveProgressMode(args.options);
 	const useProgressColor = progressMode !== "json" && resolveProgressColor(args.options);
 	let fileConfig;
-	if (configPath) try {
-		fileConfig = await loadGradingConfig(configPath);
+	const gradingConfigPath = configPath ?? suitePath;
+	if (configPath && suitePath) {
+		console.error("grade: use only one of --config or --suite");
+		return 2;
+	}
+	if (gradingConfigPath) try {
+		fileConfig = await loadGradingConfig(gradingConfigPath);
 	} catch (err) {
 		console.error(err instanceof Error ? err.message : String(err));
 		return 2;
@@ -543,7 +407,7 @@ async function gradeCommand(args) {
 			binary,
 			timeoutMs,
 			maxConcurrent
-		}, configPath);
+		}, configPath ?? suitePath);
 	} catch (err) {
 		console.error(err instanceof Error ? err.message : String(err));
 		return 2;
@@ -570,6 +434,120 @@ async function gradeCommand(args) {
 	return gradingReportPassed(grading) ? 0 : 1;
 }
 //#endregion
+//#region src/cli/commands/pipeline.ts
+/**
+* `harness-eval pipeline` — orchestrate run → grade → envelope from suite.yaml.
+*/
+/** Read package version for envelope provenance (best-effort). */
+async function readFrameworkVersion() {
+	try {
+		const text = await readFile(join(dirname(fileURLToPath(import.meta.url)), "../../../package.json"), "utf8");
+		return JSON.parse(text).version;
+	} catch {
+		return;
+	}
+}
+/** Resolve CLI path overrides relative to the suite directory unless absolute or `~/`. */
+function resolveOverridePath(value, suiteDir) {
+	if (!value) return void 0;
+	return isAbsolute(value) || value.startsWith("~/") ? value : join(suiteDir, value);
+}
+/**
+* Execute `harness-eval pipeline`.
+*
+* @returns Step exit code (0 pass, 1 eval fail, 2 usage/load error).
+*/
+async function pipelineCommand(args) {
+	const suitePath = args.positional[0];
+	if (!suitePath) {
+		console.error("usage: harness-eval pipeline <suite.yaml|dir> [--steps run,grade,envelope] [--output path] [--grading path] [--report path] [--max-concurrent N] [--progress default|quiet|verbose|json]");
+		return 2;
+	}
+	let doc;
+	try {
+		doc = await loadSuiteDocument(suitePath);
+	} catch (err) {
+		console.error(err instanceof Error ? err.message : String(err));
+		return 2;
+	}
+	if (!doc.pipeline) {
+		console.error("suite.yaml has no pipeline block; use run, grade, and envelope commands separately");
+		return 2;
+	}
+	const suiteDir = suiteDirectoryFromPath(doc.suitePath);
+	const steps = getOption(args.options, "steps");
+	const maxConcurrent = getOptionInt(args.options, "max-concurrent", 4);
+	const progressMode = resolveProgressMode(args.options);
+	const useProgressColor = progressMode !== "json" && resolveProgressColor(args.options);
+	const projection = parseEnvelopeProjection(getOption(args.options, "projection"));
+	if (getOption(args.options, "projection") && !projection) {
+		console.error("invalid --projection; expected envelope, trajectory, or instances");
+		return 2;
+	}
+	const overrides = {};
+	const runOutput = getOption(args.options, "output");
+	if (runOutput) overrides.run = {
+		output: resolveOverridePath(runOutput, suiteDir),
+		maxConcurrent
+	};
+	const reportOverride = getOption(args.options, "report");
+	if (reportOverride) {
+		overrides.grade = {
+			...overrides.grade,
+			input: resolveOverridePath(reportOverride, suiteDir)
+		};
+		overrides.envelope = {
+			...overrides.envelope,
+			report: resolveOverridePath(reportOverride, suiteDir)
+		};
+	}
+	const gradingOutput = getOption(args.options, "grading-output");
+	if (gradingOutput) overrides.grade = {
+		...overrides.grade,
+		output: resolveOverridePath(gradingOutput, suiteDir)
+	};
+	const gradingInput = getOption(args.options, "grading");
+	if (gradingInput) overrides.envelope = {
+		...overrides.envelope,
+		grading: resolveOverridePath(gradingInput, suiteDir)
+	};
+	const envelopeOutput = getOption(args.options, "envelope-output");
+	if (envelopeOutput) overrides.envelope = {
+		...overrides.envelope,
+		output: resolveOverridePath(envelopeOutput, suiteDir)
+	};
+	if (projection) overrides.envelope = {
+		...overrides.envelope,
+		projection
+	};
+	if (doc.pipeline.grade && !doc.judge) {
+		console.error("pipeline grade step requires inline judge: block in suite.yaml");
+		return 2;
+	}
+	const frameworkVersion = await readFrameworkVersion();
+	try {
+		return (await runPipeline(doc, {
+			steps,
+			maxConcurrent,
+			overrides,
+			frameworkVersion,
+			onRunProgress: createRunProgressHandler({
+				mode: progressMode,
+				maxConcurrent,
+				color: useProgressColor
+			}),
+			onGradeProgress: createGradeProgressHandler({
+				mode: progressMode,
+				maxConcurrent,
+				color: useProgressColor
+			})
+		})).exitCode;
+	} catch (err) {
+		console.error(err instanceof Error ? err.message : String(err));
+		return 2;
+	}
+}
+//#endregion
 //#region src/cli/commands/otel-output.ts
 /**
 * Write OTLP JSON artifacts from a suite report.
@@ -670,8 +648,9 @@ const USAGE = `harness-eval — harness-level eval framework
 
 Usage:
   harness-eval run <suite.yaml> [--max-concurrent N] [--baseline path] [--output path] [--otel-output dir] [--format console|markdown|json] [--adapter id] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
-  harness-eval grade <report.json> [--config grading.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N] [--format console|json] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
+  harness-eval grade <report.json> [--config grading.yaml] [--suite suite.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N] [--format console|json] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
   harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]
+  harness-eval pipeline <suite.yaml|dir> [--steps run,grade,envelope] [--output path] [--grading path] [--grading-output path] [--envelope-output path] [--report path] [--projection envelope|trajectory|instances] [--max-concurrent N] [--progress default|quiet|verbose|json]
   harness-eval format <report.json> [--format console|markdown|json] [--baseline path]
   harness-eval --help
 
@@ -698,6 +677,7 @@ async function main(argv) {
 		case "run": return await runCommand(parsed);
 		case "grade": return await gradeCommand(parsed);
 		case "envelope": return await envelopeCommand(parsed);
+		case "pipeline": return await pipelineCommand(parsed);
 		case "format": return await formatCommand(parsed);
 		case void 0:
 			console.error(USAGE);