@alis-build/harness-eval 0.1.1 → 0.1.3

package/dist/claude-code-C_7hxC8z.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-code-C_7hxC8z.js","names":[],"sources":["../src/parsers/stream-json.ts","../src/adapters/claude-code/flags.ts","../src/adapters/claude-code/process.ts","../src/adapters/claude-code/index.ts"],"sourcesContent":["/**\n * Line-buffered NDJSON parser for Claude Code's `--output-format stream-json`.\n *\n * Claude Code emits one JSON object per line on stdout. The parser:\n *   - buffers across chunk boundaries (a single JSON line may arrive in two reads)\n *   - skips empty lines (defensive — shouldn't occur, but harmless if it does)\n *   - emits a discriminated `ParseResult` per line so callers can decide whether\n *     a malformed line should abort the run or just be logged.\n *\n * Why a generator (and not a Transform stream)?\n *   The eval adapter consumes events sequentially and synchronously updates a\n *   builder. Async iteration is the simplest interface for that pattern and\n *   composes cleanly with `for await` in the adapter. A Transform would force\n *   the builder into event-handler style.\n */\n\nimport type { Readable } from \"node:stream\";\nimport type { StreamEvent } from \"../types/stream\";\n\n/**\n * Result of attempting to parse a single line.\n *\n * Successful parses yield `{ ok: true }` with the typed event and the raw line\n * (kept for diagnostics and OTel `events.attributes.raw`). Failed parses yield\n * `{ ok: false }` with the parse error and the raw line — callers can log,\n * skip, or fail the run as they see fit.\n */\nexport type ParseResult =\n  | { ok: true; event: StreamEvent; rawLine: string }\n  | { ok: false; error: Error; rawLine: string };\n\n/**\n * Parse a readable stream of NDJSON into a sequence of typed stream-json events.\n *\n * @example\n *   const child = spawn(\"claude\", [\"-p\", prompt, \"--output-format\", \"stream-json\", \"--verbose\"]);\n *   for await (const result of parseStreamJson(child.stdout)) {\n *     if (result.ok) builder.consume(result.event);\n *     else console.warn(\"malformed stream line:\", result.rawLine, result.error);\n *   }\n */\nexport async function* parseStreamJson(\n  stream: Readable,\n): AsyncGenerator<ParseResult, void, void> {\n  let buffer = \"\";\n  // The Node child_process stdout is a binary stream by default. Setting the\n  // encoding here means `for await (const chunk of stream)` yields strings.\n  stream.setEncoding(\"utf8\");\n\n  for await (const chunk of stream) {\n    buffer += chunk as string;\n\n    // Drain every complete line currently in the buffer before reading more.\n    // Multiple JSON objects can arrive in one chunk (e.g. when the harness\n    // emits a burst of events at session start).\n    let newlineIdx: number;\n    while ((newlineIdx = buffer.indexOf(\"\\n\")) !== -1) {\n      const line = buffer.slice(0, newlineIdx).trim();\n      buffer = buffer.slice(newlineIdx + 1);\n      if (line.length === 0) continue;\n      yield tryParseLine(line);\n    }\n  }\n\n  // Flush any trailing content that arrived without a final newline. Stream-json\n  // typically ends with a newline-terminated `result` event, but a killed\n  // process may not flush, so we still try to emit what we have.\n  const trailing = buffer.trim();\n  if (trailing.length > 0) {\n    yield tryParseLine(trailing);\n  }\n}\n\n/**\n * Parse a single line. Extracted as a helper so the generator stays readable.\n *\n * Note: we do not validate the event structure beyond `JSON.parse`. Runtime\n * validation (e.g. zod) is overkill here — the schema is stable enough at\n * runtime, and the TrajectoryBuilder is tolerant of missing fields. Adding\n * validation would be premature.\n */\nfunction tryParseLine(line: string): ParseResult {\n  try {\n    const event = JSON.parse(line) as StreamEvent;\n    return { ok: true, event, rawLine: line };\n  } catch (err) {\n    return {\n      ok: false,\n      error: err instanceof Error ? err : new Error(String(err)),\n      rawLine: line,\n    };\n  }\n}\n","/**\n * Build CLI args for Claude Code judge subprocesses (JSON output, not stream-json).\n *\n * Shared flag assembly for harness runs (`buildArgs`) and LLM grading judges\n * (`buildJudgeArgs`).\n */\n\nimport type { ClaudeCodeAdapterConfig, ClaudeCodeOptions } from \"./types\";\n\n/** Append repeated `--flag value` pairs for array config fields. */\nfunction pushRepeatableFlag(args: string[], flag: string, values?: string[]): void {\n  if (!values) return;\n  for (const value of values) {\n    args.push(flag, value);\n  }\n}\n\n/**\n * Append an optional CLI flag. Boolean `true` emits the flag alone; other\n * scalars emit `--flag value`.\n */\nfunction pushOptionalFlag(\n  args: string[],\n  flag: string,\n  value: string | number | boolean | undefined,\n): void {\n  if (value === undefined) return;\n  if (typeof value === \"boolean\") {\n    if (value) args.push(flag);\n    return;\n  }\n  args.push(flag, String(value));\n}\n\n/** Append Claude Code CLI flags shared by harness runs and grading judges. */\nexport function appendClaudeCodeFlags(\n  args: string[],\n  config: ClaudeCodeOptions & { model?: string },\n): void {\n  pushRepeatableFlag(args, \"--plugin-dir\", config.pluginDirs);\n  pushRepeatableFlag(args, \"--plugin-url\", config.pluginUrls);\n  pushRepeatableFlag(args, \"--add-dir\", config.addDirs);\n\n  pushOptionalFlag(args, \"--mcp-config\", config.mcpConfig);\n  pushOptionalFlag(args, \"--model\", config.model);\n  pushOptionalFlag(args, \"--permission-mode\", config.permissionMode);\n  pushOptionalFlag(args, \"--effort\", config.effort);\n  pushOptionalFlag(args, \"--agent\", config.agent);\n  pushOptionalFlag(args, \"--fallback-model\", config.fallbackModel);\n  pushOptionalFlag(args, \"--tools\", config.tools);\n  pushOptionalFlag(args, \"--settings\", config.settings);\n  pushOptionalFlag(args, \"--setting-sources\", config.settingSources);\n  pushOptionalFlag(args, \"--max-turns\", config.maxTurns);\n  pushOptionalFlag(args, \"--max-budget-usd\", config.maxBudgetUsd);\n  pushOptionalFlag(args, \"--system-prompt\", config.systemPrompt);\n  pushOptionalFlag(args, \"--system-prompt-file\", config.systemPromptFile);\n  pushOptionalFlag(args, \"--append-system-prompt\", config.appendSystemPrompt);\n  pushOptionalFlag(\n    args,\n    \"--append-system-prompt-file\",\n    config.appendSystemPromptFile,\n  );\n  pushOptionalFlag(args, \"--debug\", config.debug);\n  pushOptionalFlag(args, \"--debug-file\", config.debugFile);\n\n  if (config.allowedTools && config.allowedTools.length > 0) {\n    args.push(\"--allowedTools\", config.allowedTools.join(\",\"));\n  }\n\n  if (config.disallowedTools && config.disallowedTools.length > 0) {\n    args.push(\"--disallowedTools\", config.disallowedTools.join(\",\"));\n  }\n\n  pushOptionalFlag(args, \"--strict-mcp-config\", config.strictMcpConfig);\n  pushOptionalFlag(args, \"--include-hook-events\", config.includeHookEvents);\n  pushOptionalFlag(args, \"--no-session-persistence\", config.noSessionPersistence);\n  pushOptionalFlag(args, \"--disable-slash-commands\", config.disableSlashCommands);\n  pushOptionalFlag(args, \"--bare\", config.bare);\n  pushOptionalFlag(args, \"--safe-mode\", config.safeMode);\n  pushOptionalFlag(\n    args,\n    \"--allow-dangerously-skip-permissions\",\n    config.allowDangerouslySkipPermissions,\n  );\n  pushOptionalFlag(\n    args,\n    \"--dangerously-skip-permissions\",\n    config.dangerouslySkipPermissions,\n  );\n}\n\n/**\n * Build the argument vector for spawning `claude`.\n *\n * Order matters only for flags that take values — value flags must come\n * after their flag name. Everything else is order-independent.\n */\nexport function buildArgs(config: ClaudeCodeAdapterConfig): string[] {\n  const args: string[] = [\n    \"-p\",\n    config.prompt,\n    \"--output-format\",\n    \"stream-json\",\n    \"--verbose\",\n  ];\n\n  appendClaudeCodeFlags(args, config);\n\n  return args;\n}\n\n/**\n * Build args for an LLM judge subprocess (`--output-format json`).\n *\n * Defaults permission mode to `bypassPermissions` so the judge does not\n * block on tool permission prompts during single-shot JSON grading.\n */\nexport function buildJudgeArgs(\n  prompt: string,\n  config: ClaudeCodeOptions & { model?: string } = {},\n): string[] {\n  const args: string[] = [\"-p\", prompt, \"--output-format\", \"json\"];\n  const permissionMode = config.permissionMode ?? \"bypassPermissions\";\n  appendClaudeCodeFlags(args, {\n    ...config,\n    permissionMode,\n  });\n  return args;\n}\n","/**\n * Process management for the Claude Code adapter.\n *\n * This module owns spawning, timeout, abort signal handling, and process-tree\n * teardown. The orchestrator (`index.ts`) consumes the returned handle —\n * reading stdout and waiting for completion — but doesn't worry about how\n * the process gets killed or how its config gets isolated.\n *\n * Why a separate module? Process management is the one part of the adapter\n * with real I/O complexity (process groups, signal escalation, temp-dir\n * lifecycle, env merging). Isolating it makes the orchestrator easy to read\n * and lets us swap the spawning logic if we later need to, e.g., wrap claude\n * in a sandbox runner.\n */\n\nimport { spawn, type ChildProcess } from \"node:child_process\";\nimport { mkdtemp, rm } from \"node:fs/promises\";\nimport { tmpdir } from \"node:os\";\nimport { join } from \"node:path\";\nimport type { Readable } from \"node:stream\";\n\nimport { buildArgs } from \"./flags\";\nimport type { ClaudeCodeAdapterConfig } from \"./types\";\n\n/** Default hard timeout per run. Tunable via config.timeoutMs. */\nconst DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;\n\n/**\n * Grace period between SIGTERM and SIGKILL. Most processes shut down cleanly\n * within a few seconds; this gives them that chance while preventing CI from\n * hanging indefinitely on a stuck child.\n */\nconst KILL_GRACE_MS = 5_000;\n\n/**\n * Handle to a spawned `claude` process. The orchestrator drives it:\n *   - Read `stdout` (typically via parseStreamJson).\n *   - Await `done` to learn the exit state.\n *   - Await `stderrCollected` for diagnostic stderr.\n *   - Check `timedOut()` after exit to distinguish kill-by-timeout from\n *     normal termination.\n *   - Call `cleanup()` after all of the above to remove the temp config dir.\n */\nexport interface SpawnedClaude {\n  stdout: Readable;\n  done: Promise<{ exitCode: number | null; signal: NodeJS.Signals | null }>;\n  stderrCollected: Promise<string>;\n  timedOut: () => boolean;\n  cleanup: () => Promise<void>;\n}\n\n/**\n * Spawn `claude` in headless mode with isolated config and a process-group\n * lifecycle. See {@link SpawnedClaude} for how to consume the result.\n *\n * **Kill sequence:** timeout and abort both follow the same two-step path:\n * `SIGTERM` to the process group, then `SIGKILL` after {@link KILL_GRACE_MS}\n * if the group is still alive. This avoids leaving MCP/tool subprocesses\n * running while still giving claude a chance to flush stream-json output.\n *\n * @param config - Adapter options; `timeoutMs`, `signal`, and `isolateConfig`\n *   control lifecycle and config isolation.\n */\nexport async function spawnClaude(\n  config: ClaudeCodeAdapterConfig,\n): Promise<SpawnedClaude> {\n  const binary = config.binary ?? \"claude\";\n  const args = buildArgs(config);\n\n  const isolateConfig = config.isolateConfig !== false;\n\n  // Isolated runs use a fresh temp dir so plugins/settings don't leak between\n  // reps. Non-isolated runs inherit the caller's Claude login and plugins.\n  const tempConfigDir = isolateConfig\n    ? await mkdtemp(join(tmpdir(), \"harness-eval-\"))\n    : null;\n\n  const env: Record<string, string | undefined> = {\n    ...process.env,\n    ...config.env,\n  };\n  if (tempConfigDir) {\n    // Override after ...env so callers can't accidentally un-isolate.\n    env.CLAUDE_CONFIG_DIR = tempConfigDir;\n  }\n\n  const child = spawn(binary, args, {\n    cwd: config.cwd ?? process.cwd(),\n    env,\n    stdio: [\"ignore\", \"pipe\", \"pipe\"],\n    // detached: true means the child becomes the leader of its own process\n    // group. We exploit this to kill the entire group (including any MCP\n    // server subprocesses and tool processes) on timeout/abort.\n    detached: true,\n  });\n\n\n  // `timedOut` is set only by the hard timeout timer, not by abort — callers\n  // use it to distinguish \"ran too long\" from user cancellation or normal exit.\n  let timedOut = false;\n  let killEscalation: NodeJS.Timeout | null = null;\n  const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n\n  /**\n   * Arm (or re-arm) the SIGKILL fallback. Each SIGTERM attempt gets its own\n   * grace window so a slow shutdown doesn't leave orphaned MCP servers.\n   */\n  const scheduleKillEscalation = () => {\n    if (killEscalation) clearTimeout(killEscalation);\n    killEscalation = setTimeout(\n      () => killTree(child, \"SIGKILL\"),\n      KILL_GRACE_MS,\n    );\n  };\n\n  const timeoutTimer = setTimeout(() => {\n    timedOut = true;\n    killTree(child, \"SIGTERM\");\n    scheduleKillEscalation();\n  }, timeoutMs);\n\n  // AbortSignal cancellation mirrors timeout kills but does not flip `timedOut`.\n  const onAbort = () => {\n    killTree(child, \"SIGTERM\");\n    scheduleKillEscalation();\n  };\n  config.signal?.addEventListener(\"abort\", onAbort, { once: true });\n\n\n  // Drain stderr eagerly so the OS-level buffer never fills and stalls the\n  // child (Node child processes will block on a full pipe).\n  const stderrChunks: string[] = [];\n  child.stderr?.setEncoding(\"utf8\");\n  child.stderr?.on(\"data\", (chunk: string) => {\n    stderrChunks.push(chunk);\n  });\n\n  const stderrCollected = new Promise<string>((resolve) => {\n    const finalize = () => resolve(stderrChunks.join(\"\"));\n    child.stderr?.on(\"end\", finalize);\n    // Errors during stderr capture shouldn't fail the whole run; we just\n    // return what we've buffered so far.\n    child.stderr?.on(\"error\", finalize);\n  });\n\n\n  // Resolve once the process exits or fails to spawn. Guard against double\n  // settlement because both `close` and `error` can fire in edge cases.\n  const done = new Promise<{\n    exitCode: number | null;\n    signal: NodeJS.Signals | null;\n  }>((resolve) => {\n    let settled = false;\n    const finalize = (\n      exitCode: number | null,\n      signal: NodeJS.Signals | null,\n    ) => {\n      if (settled) return;\n      settled = true;\n      // Tear down timers/listeners so a late timeout cannot SIGKILL a reused PID.\n      clearTimeout(timeoutTimer);\n      if (killEscalation) clearTimeout(killEscalation);\n      config.signal?.removeEventListener(\"abort\", onAbort);\n      resolve({ exitCode, signal });\n    };\n\n    child.on(\"close\", (code, signal) => finalize(code, signal));\n    // ENOENT and other spawn failures emit `error` — `close` may not follow.\n    child.on(\"error\", () => finalize(null, null));\n  });\n\n\n  const cleanup = async () => {\n    if (!tempConfigDir) return;\n    try {\n      await rm(tempConfigDir, { recursive: true, force: true });\n    } catch {\n      // Best-effort. A leftover temp dir is annoying but not catastrophic;\n      // we don't want to fail the run for it.\n    }\n  };\n\n  // stdout is guaranteed non-null because we passed `stdio: [..., \"pipe\", ...]`.\n  // The `!` is safe; the alternative would be a redundant runtime check that\n  // could never fire.\n  return {\n    stdout: child.stdout!,\n    done,\n    stderrCollected,\n    timedOut: () => timedOut,\n    cleanup,\n  };\n}\n\n/**\n * Kill the child's process group, then fall back to the bare PID if the\n * group is already gone. This catches MCP server subprocesses and tool\n * processes spawned by claude.\n *\n * **Signal escalation:** callers typically invoke this first with `SIGTERM`,\n * then again with `SIGKILL` after {@link KILL_GRACE_MS}. The group kill is\n * essential — a bare `child.kill()` would leave MCP servers running.\n *\n * **Platform edge case:** when the group leader exits first, `kill(-pid)`\n * throws `ESRCH`. The single-PID fallback covers that without failing the\n * adapter run.\n *\n * @param child - Spawned process handle from {@link spawn}.\n * @param signal - POSIX signal to deliver (`SIGTERM` or `SIGKILL` in practice).\n */\nfunction killTree(child: ChildProcess, signal: NodeJS.Signals): void {\n  if (child.pid === undefined) return;\n  try {\n    // Negative PID targets the entire process group (requires detached spawn).\n    process.kill(-child.pid, signal);\n  } catch {\n    try {\n      // Group already reaped — try the leader PID directly.\n      child.kill(signal);\n    } catch {\n      // Process fully gone; nothing to do.\n    }\n  }\n}\n","/**\n * Claude Code adapter — public API.\n */\n\nimport { parseStreamJson } from \"../../parsers/stream-json\";\nimport { TrajectoryBuilder } from \"../../trajectory/builder\";\nimport type { StreamEvent } from \"../../types/stream\";\n\nimport { AdapterError } from \"../types\";\nimport { spawnClaude } from \"./process\";\nimport type {\n  AdapterDiagnostics,\n  ClaudeCodeAdapterConfig,\n  ClaudeCodeAdapterResult,\n  ParseErrorRecord,\n} from \"./types\";\nimport type { HarnessAdapter } from \"../types\";\n\nexport { AdapterError } from \"../types\";\nexport type {\n  AdapterDiagnostics,\n  AdapterResult,\n  ClaudeCodeAdapterConfig,\n  ClaudeCodeAdapterResult,\n  ClaudeCodeOptions,\n  ParseErrorRecord,\n  PermissionMode,\n} from \"./types\";\n\n/**\n * Run Claude Code in headless mode and return a trajectory.\n */\nexport async function runClaudeCode(\n  config: ClaudeCodeAdapterConfig,\n): Promise<ClaudeCodeAdapterResult> {\n  const startTs = Date.now();\n  const spawned = await spawnClaude(config);\n\n  const builder = new TrajectoryBuilder();\n  const rawEvents: StreamEvent[] = [];\n  const parseErrors: ParseErrorRecord[] = [];\n\n  try {\n    for await (const result of parseStreamJson(spawned.stdout)) {\n      if (result.ok) {\n        builder.consume(result.event);\n        rawEvents.push(result.event);\n      } else {\n        parseErrors.push({\n          line: result.rawLine,\n          error: result.error.message,\n        });\n      }\n    }\n\n    const [{ exitCode, signal }, stderr] = await Promise.all([\n      spawned.done,\n      spawned.stderrCollected,\n    ]);\n\n    const diagnostics: AdapterDiagnostics = {\n      exitCode,\n      signal,\n      stderr,\n      parseErrors,\n      timedOut: spawned.timedOut(),\n      durationMs: Date.now() - startTs,\n    };\n\n    let view;\n    try {\n      view = builder.build();\n    } catch (err) {\n      const message = err instanceof Error ? err.message : String(err);\n      throw new AdapterError(\n        `harness produced no usable trajectory: ${message}`,\n        diagnostics,\n      );\n    }\n\n    return { view, diagnostics, rawEvents };\n  } finally {\n    await spawned.cleanup();\n  }\n}\n\n/** Registered {@link HarnessAdapter} for Claude Code headless runs. */\nexport const claudeCodeAdapter: HarnessAdapter<ClaudeCodeAdapterConfig> = {\n  id: \"claude-code\",\n  run: runClaudeCode,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;AAyCA,gBAAuB,gBACrB,QACyC;CACzC,IAAI,SAAS;CAGb,OAAO,YAAY,MAAM;CAEzB,WAAW,MAAM,SAAS,QAAQ;EAChC,UAAU;EAKV,IAAI;EACJ,QAAQ,aAAa,OAAO,QAAQ,IAAI,OAAO,IAAI;GACjD,MAAM,OAAO,OAAO,MAAM,GAAG,UAAU,CAAC,CAAC,KAAK;GAC9C,SAAS,OAAO,MAAM,aAAa,CAAC;GACpC,IAAI,KAAK,WAAW,GAAG;GACvB,MAAM,aAAa,IAAI;EACzB;CACF;CAKA,MAAM,WAAW,OAAO,KAAK;CAC7B,IAAI,SAAS,SAAS,GACpB,MAAM,aAAa,QAAQ;AAE/B;;;;;;;;;AAUA,SAAS,aAAa,MAA2B;CAC/C,IAAI;EAEF,OAAO;GAAE,IAAI;GAAM,OADL,KAAK,MAAM,IACF;GAAG,SAAS;EAAK;CAC1C,SAAS,KAAK;EACZ,OAAO;GACL,IAAI;GACJ,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;GACzD,SAAS;EACX;CACF;AACF;;;;AClFA,SAAS,mBAAmB,MAAgB,MAAc,QAAyB;CACjF,IAAI,CAAC,QAAQ;CACb,KAAK,MAAM,SAAS,QAClB,KAAK,KAAK,MAAM,KAAK;AAEzB;;;;;AAMA,SAAS,iBACP,MACA,MACA,OACM;CACN,IAAI,UAAU,KAAA,GAAW;CACzB,IAAI,OAAO,UAAU,WAAW;EAC9B,IAAI,OAAO,KAAK,KAAK,IAAI;EACzB;CACF;CACA,KAAK,KAAK,MAAM,OAAO,KAAK,CAAC;AAC/B;;AAGA,SAAgB,sBACd,MACA,QACM;CACN,mBAAmB,MAAM,gBAAgB,OAAO,UAAU;CAC1D,mBAAmB,MAAM,gBAAgB,OAAO,UAAU;CAC1D,mBAAmB,MAAM,aAAa,OAAO,OAAO;CAEpD,iBAAiB,MAAM,gBAAgB,OAAO,SAAS;CACvD,iBAAiB,MAAM,WAAW,OAAO,KAAK;CAC9C,iBAAiB,MAAM,qBAAqB,OAAO,cAAc;CACjE,iBAAiB,MAAM,YAAY,OAAO,MAAM;CAChD,iBAAiB,MAAM,WAAW,OAAO,KAAK;CAC9C,iBAAiB,MAAM,oBAAoB,OAAO,aAAa;CAC/D,iBAAiB,MAAM,WAAW,OAAO,KAAK;CAC9C,iBAAiB,MAAM,cAAc,OAAO,QAAQ;CACpD,iBAAiB,MAAM,qBAAqB,OAAO,cAAc;CACjE,iBAAiB,MAAM,eAAe,OAAO,QAAQ;CACrD,iBAAiB,MAAM,oBAAoB,OAAO,YAAY;CAC9D,iBAAiB,MAAM,mBAAmB,OAAO,YAAY;CAC7D,iBAAiB,MAAM,wBAAwB,OAAO,gBAAgB;CACtE,iBAAiB,MAAM,0BAA0B,OAAO,kBAAkB;CAC1E,iBACE,MACA,+BACA,OAAO,sBACT;CACA,iBAAiB,MAAM,WAAW,OAAO,KAAK;CAC9C,iBAAiB,MAAM,gBAAgB,OAAO,SAAS;CAEvD,IAAI,OAAO,gBAAgB,OAAO,aAAa,SAAS,GACtD,KAAK,KAAK,kBAAkB,OAAO,aAAa,KAAK,GAAG,CAAC;CAG3D,IAAI,OAAO,mBAAmB,OAAO,gBAAgB,SAAS,GAC5D,KAAK,KAAK,qBAAqB,OAAO,gBAAgB,KAAK,GAAG,CAAC;CAGjE,iBAAiB,MAAM,uBAAuB,OAAO,eAAe;CACpE,iBAAiB,MAAM,yBAAyB,OAAO,iBAAiB;CACxE,iBAAiB,MAAM,4BAA4B,OAAO,oBAAoB;CAC9E,iBAAiB,MAAM,4BAA4B,OAAO,oBAAoB;CAC9E,iBAAiB,MAAM,UAAU,OAAO,IAAI;CAC5C,iBAAiB,MAAM,eAAe,OAAO,QAAQ;CACrD,iBACE,MACA,wCACA,OAAO,+BACT;CACA,iBACE,MACA,kCACA,OAAO,0BACT;AACF;;;;;;;AAQA,SAAgB,UAAU,QAA2C;CACnE,MAAM,OAAiB;EACrB;EACA,OAAO;EACP;EACA;EACA;CACF;CAEA,sBAAsB,MAAM,MAAM;CAElC,OAAO;AACT;;;;;;;AAQA,SAAgB,eACd,QACA,SAAiD,CAAC,GACxC;CACV,MAAM,OAAiB;EAAC;EAAM;EAAQ;EAAmB;CAAM;CAC/D,MAAM,iBAAiB,OAAO,kBAAkB;CAChD,sBAAsB,MAAM;EAC1B,GAAG;EACH;CACF,CAAC;CACD,OAAO;AACT;;;;;;;;;;;;;;;;;;ACvGA,MAAM,qBAAqB,MAAS;;;;;;AAOpC,MAAM,gBAAgB;;;;;;;;;;;;;AA+BtB,eAAsB,YACpB,QACwB;CACxB,MAAM,SAAS,OAAO,UAAU;CAChC,MAAM,OAAO,UAAU,MAAM;CAM7B,MAAM,gBAJgB,OAAO,kBAAkB,QAK3C,MAAM,QAAQ,KAAK,OAAO,GAAG,eAAe,CAAC,IAC7C;CAEJ,MAAM,MAA0C;EAC9C,GAAG,QAAQ;EACX,GAAG,OAAO;CACZ;CACA,IAAI,eAEF,IAAI,oBAAoB;CAG1B,MAAM,QAAQ,MAAM,QAAQ,MAAM;EAChC,KAAK,OAAO,OAAO,QAAQ,IAAI;EAC/B;EACA,OAAO;GAAC;GAAU;GAAQ;EAAM;EAIhC,UAAU;CACZ,CAAC;CAKD,IAAI,WAAW;CACf,IAAI,iBAAwC;CAC5C,MAAM,YAAY,OAAO,aAAa;;;;;CAMtC,MAAM,+BAA+B;EACnC,IAAI,gBAAgB,aAAa,cAAc;EAC/C,iBAAiB,iBACT,SAAS,OAAO,SAAS,GAC/B,aACF;CACF;CAEA,MAAM,eAAe,iBAAiB;EACpC,WAAW;EACX,SAAS,OAAO,SAAS;EACzB,uBAAuB;CACzB,GAAG,SAAS;CAGZ,MAAM,gBAAgB;EACpB,SAAS,OAAO,SAAS;EACzB,uBAAuB;CACzB;CACA,OAAO,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAKhE,MAAM,eAAyB,CAAC;CAChC,MAAM,QAAQ,YAAY,MAAM;CAChC,MAAM,QAAQ,GAAG,SAAS,UAAkB;EAC1C,aAAa,KAAK,KAAK;CACzB,CAAC;CAED,MAAM,kBAAkB,IAAI,SAAiB,YAAY;EACvD,MAAM,iBAAiB,QAAQ,aAAa,KAAK,EAAE,CAAC;EACpD,MAAM,QAAQ,GAAG,OAAO,QAAQ;EAGhC,MAAM,QAAQ,GAAG,SAAS,QAAQ;CACpC,CAAC;CAKD,MAAM,OAAO,IAAI,SAGb,YAAY;EACd,IAAI,UAAU;EACd,MAAM,YACJ,UACA,WACG;GACH,IAAI,SAAS;GACb,UAAU;GAEV,aAAa,YAAY;GACzB,IAAI,gBAAgB,aAAa,cAAc;GAC/C,OAAO,QAAQ,oBAAoB,SAAS,OAAO;GACnD,QAAQ;IAAE;IAAU;GAAO,CAAC;EAC9B;EAEA,MAAM,GAAG,UAAU,MAAM,WAAW,SAAS,MAAM,MAAM,CAAC;EAE1D,MAAM,GAAG,eAAe,SAAS,MAAM,IAAI,CAAC;CAC9C,CAAC;CAGD,MAAM,UAAU,YAAY;EAC1B,IAAI,CAAC,eAAe;EACpB,IAAI;GACF,MAAM,GAAG,eAAe;IAAE,WAAW;IAAM,OAAO;GAAK,CAAC;EAC1D,QAAQ,CAGR;CACF;CAKA,OAAO;EACL,QAAQ,MAAM;EACd;EACA;EACA,gBAAgB;EAChB;CACF;AACF;;;;;;;;;;;;;;;;;AAkBA,SAAS,SAAS,OAAqB,QAA8B;CACnE,IAAI,MAAM,QAAQ,KAAA,GAAW;CAC7B,IAAI;EAEF,QAAQ,KAAK,CAAC,MAAM,KAAK,MAAM;CACjC,QAAQ;EACN,IAAI;GAEF,MAAM,KAAK,MAAM;EACnB,QAAQ,CAER;CACF;AACF;;;;;;;;;;;;;;AC/LA,eAAsB,cACpB,QACkC;CAClC,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,UAAU,MAAM,YAAY,MAAM;CAExC,MAAM,UAAU,IAAI,kBAAkB;CACtC,MAAM,YAA2B,CAAC;CAClC,MAAM,cAAkC,CAAC;CAEzC,IAAI;EACF,WAAW,MAAM,UAAU,gBAAgB,QAAQ,MAAM,GACvD,IAAI,OAAO,IAAI;GACb,QAAQ,QAAQ,OAAO,KAAK;GAC5B,UAAU,KAAK,OAAO,KAAK;EAC7B,OACE,YAAY,KAAK;GACf,MAAM,OAAO;GACb,OAAO,OAAO,MAAM;EACtB,CAAC;EAIL,MAAM,CAAC,EAAE,UAAU,UAAU,UAAU,MAAM,QAAQ,IAAI,CACvD,QAAQ,MACR,QAAQ,eACV,CAAC;EAED,MAAM,cAAkC;GACtC;GACA;GACA;GACA;GACA,UAAU,QAAQ,SAAS;GAC3B,YAAY,KAAK,IAAI,IAAI;EAC3B;EAEA,IAAI;EACJ,IAAI;GACF,OAAO,QAAQ,MAAM;EACvB,SAAS,KAAK;GAEZ,MAAM,IAAI,aACR,0CAFc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,KAG7D,WACF;EACF;EAEA,OAAO;GAAE;GAAM;GAAa;EAAU;CACxC,UAAU;EACR,MAAM,QAAQ,QAAQ;CACxB;AACF;;AAGA,MAAa,oBAA6D;CACxE,IAAI;CACJ,KAAK;AACP"}

package/dist/cli/bin.js

@@ -1,129 +1,19 @@
 #!/usr/bin/env node
-import { C as gradeReport, O as trajectoryToOtlp, S as resolveGradeOptions, a as toProtoInstances, b as formatGradingConsole, i as toAgentTrace, n as buildEvalRunEnvelopeFromFiles, o as toTrajectory, w as loadSuiteReport, x as gradingReportPassed, y as formatReport } from "../build-DsVJ_UeU.js";
-import { t as runSuite, u as getAdapter } from "../suite-chj0j22j.js";
-import { i as loadGradingConfig, t as loadSuite } from "../loader-BCnFJ8rm.js";
+import { F as loadSuiteReport, M as gradingReportPassed, N as resolveGradeOptions, P as gradeReport, a as envelopeCommand, c as getOptionInt, i as runPipeline, j as formatGradingConsole, l as hasOption, o as parseEnvelopeProjection, p as suiteDirectoryFromPath, r as trajectoryToOtlp, s as getOption, t as formatReport, u as parseArgs } from "../reporter-Biy-5-9M.js";
+import { t as runSuite, u as getAdapter } from "../suite-BcP64nlb.js";
+import { i as loadGradingConfig, o as loadSuiteDocument, t as loadSuite } from "../loader-DnQ6Jt0i.js";
 import { mkdir, readFile, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
+import { dirname, isAbsolute, join } from "node:path";
 import { fileURLToPath } from "node:url";
-//#region src/cli/args.ts
-function parseArgs(argv) {
-	const positional = [];
-	const options = {};
-	let command;
-	const args = [...argv];
-	if (args.length > 0 && !args[0].startsWith("-")) command = args.shift();
-	for (let i = 0; i < args.length; i++) {
-		const arg = args[i];
-		if (arg === "--") {
-			positional.push(...args.slice(i + 1));
-			break;
-		}
-		if (arg.startsWith("--")) {
-			const key = arg.slice(2);
-			const next = args[i + 1];
-			if (next && !next.startsWith("-")) {
-				options[key] = next;
-				i++;
-			} else options[key] = true;
-		} else if (arg.startsWith("-") && arg.length === 2) {
-			const key = arg.slice(1);
-			const next = args[i + 1];
-			if (next && !next.startsWith("-")) {
-				options[key] = next;
-				i++;
-			} else options[key] = true;
-		} else positional.push(arg);
-	}
-	return {
-		command,
-		positional,
-		options
-	};
-}
-function getOption(options, name) {
-	const v = options[name];
-	return typeof v === "string" ? v : void 0;
-}
-function getOptionInt(options, name, defaultValue) {
-	const v = getOption(options, name);
-	if (v === void 0) return defaultValue;
-	const n = Number.parseInt(v, 10);
-	if (!Number.isFinite(n)) return defaultValue;
-	return n;
-}
-function hasOption(options, name) {
-	const v = options[name];
-	return v === true || typeof v === "string" && v === "true";
-}
-//#endregion
-//#region src/cli/commands/envelope.ts
-/**
-* `harness-eval envelope` — build EvalRunEnvelope and interchange projections.
-*/
-const PROJECTIONS = /* @__PURE__ */ new Set([
-	"envelope",
-	"trajectory",
-	"instances",
-	"agent-trace"
-]);
-function parseEnvelopeProjection(value) {
-	if (value === void 0) return "envelope";
-	if (PROJECTIONS.has(value)) return value;
-}
-function serializeEnvelopeProjection(envelope, projection) {
-	switch (projection) {
-		case "trajectory": return `${toTrajectory(envelope).map((row) => JSON.stringify(row)).join("\n")}\n`;
-		case "instances": return `${JSON.stringify(toProtoInstances(envelope), null, 2)}\n`;
-		case "agent-trace": return `${JSON.stringify(toAgentTrace(envelope), null, 2)}\n`;
-		default: return `${JSON.stringify(envelope, null, 2)}\n`;
-	}
-}
-async function readFrameworkVersion() {
-	try {
-		const text = await readFile(join(dirname(fileURLToPath(import.meta.url)), "../../../package.json"), "utf8");
-		return JSON.parse(text).version;
-	} catch {
-		return;
-	}
-}
-async function envelopeCommand(args) {
-	const reportPath = args.positional[0];
-	if (!reportPath) {
-		console.error("usage: harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances|agent-trace] [--include-raw-stream-events] [--no-transcript]");
-		return 2;
-	}
-	const outputPath = getOption(args.options, "output");
-	const gradingPath = getOption(args.options, "grading");
-	const suitePath = getOption(args.options, "suite");
-	const projection = parseEnvelopeProjection(getOption(args.options, "projection"));
-	if (!projection) {
-		console.error("invalid --projection; expected envelope, trajectory, instances, or agent-trace");
-		return 2;
-	}
-	let envelope;
-	try {
-		const frameworkVersion = await readFrameworkVersion();
-		envelope = await buildEvalRunEnvelopeFromFiles(reportPath, {
-			gradingPath,
-			suitePath,
-			includeTranscript: !hasOption(args.options, "no-transcript"),
-			includeRawStreamEvents: hasOption(args.options, "include-raw-stream-events"),
-			harness: { frameworkVersion }
-		});
-	} catch (err) {
-		console.error(err instanceof Error ? err.message : String(err));
-		return 2;
-	}
-	const serialized = serializeEnvelopeProjection(envelope, projection);
-	if (outputPath) await writeFile(outputPath, serialized, "utf8");
-	else process.stdout.write(serialized);
-	return envelope.summary.behavioralPass ? 0 : 1;
-}
-//#endregion
 //#region src/cli/commands/format.ts
 /**
 * `harness-eval format` command.
 */
+/**
+* Execute `harness-eval format`: re-render a saved report JSON.
+*
+* @returns 0 when all cells pass, 1 otherwise, 2 on load errors.
+*/
 async function formatCommand(args) {
 	const reportPath = args.positional[0];
 	if (!reportPath) {
@@ -152,11 +42,17 @@ async function formatCommand(args) {
 }
 //#endregion
 //#region src/cli/progress.ts
+/** ANSI SGR codes for progress output. Disabled when {@link resolveProgressColor} returns false. */
 const GREEN = "\x1B[32m";
 const RED = "\x1B[31m";
 const YELLOW = "\x1B[33m";
 const DIM = "\x1B[2m";
 const RESET = "\x1B[0m";
+/**
+* Resolve progress mode from `--progress`, `--quiet`, or `--verbose` flags.
+*
+* Explicit `--progress` wins; otherwise `--quiet` / `--verbose` map to modes.
+*/
 function resolveProgressMode(options) {
 	const progress = getOption(options, "progress");
 	if (progress === "json" || progress === "quiet" || progress === "verbose" || progress === "default") return progress;
@@ -164,7 +60,12 @@ function resolveProgressMode(options) {
 	if (hasOption(options, "verbose")) return "verbose";
 	return "default";
 }
-/** Whether to emit ANSI colors on the progress stream (stderr). */
+/**
+* Whether to emit ANSI colors on the progress stream (stderr).
+*
+* Precedence: `--no-color` → off; `--color` → on; `NO_COLOR` env → off;
+* `FORCE_COLOR` (non-zero) → on; otherwise TTY detection on `stream`.
+*/
 function resolveProgressColor(options, stream = process.stderr) {
 	if (hasOption(options, "no-color")) return false;
 	if (hasOption(options, "color")) return true;
@@ -172,24 +73,35 @@ function resolveProgressColor(options, stream = process.stderr) {
 	if (process.env.FORCE_COLOR !== void 0 && process.env.FORCE_COLOR !== "0") return true;
 	return "isTTY" in stream && stream.isTTY === true;
 }
+/** Green checkmark prefix for per-rep success lines. */
 function okMark(color) {
 	return color ? `${GREEN}✓${RESET}` : "✓";
 }
+/** Red cross prefix for per-rep failure lines. */
 function failMark(color) {
 	return color ? `${RED}✗${RESET}` : "✗";
 }
+/** Inline lowercase status word for repetition rows. */
 function okStatus(color) {
 	return color ? `${GREEN}ok${RESET}` : "ok";
 }
+/** Inline uppercase status word for repetition failures. */
 function failStatus(color) {
 	return color ? `${RED}FAIL${RESET}` : "FAIL";
 }
+/** Uppercase cell-level pass label in {@link formatCellSummary}. */
 function passLabel(color) {
 	return color ? `${GREEN}PASS${RESET}` : "PASS";
 }
+/** Uppercase cell-level fail label in {@link formatCellSummary}. */
 function failLabel(color) {
 	return color ? `${RED}FAIL${RESET}` : "FAIL";
 }
+/**
+* Build a {@link ProgressCallback} for suite runs.
+*
+* Writes to `options.stream` (default stderr). JSON mode emits one event per line.
+*/
 function createRunProgressHandler(options) {
 	const stream = options.stream ?? process.stderr;
 	const mode = options.mode;
@@ -295,6 +207,7 @@ function createRunProgressHandler(options) {
 		}
 	};
 }
+/** Build a progress handler for outcome grading ({@link GradeProgressEvent}). */
 function createGradeProgressHandler(options) {
 	const stream = options.stream ?? process.stderr;
 	const mode = options.mode;
@@ -375,9 +288,16 @@ function createGradeProgressHandler(options) {
 		}
 	};
 }
+/**
+* Write one NDJSON progress event line to the progress stream.
+*
+* JSON mode keeps stdout clean for machine-readable reports while still
+* exposing structured progress for CI log parsers.
+*/
 function writeJson(stream, value) {
 	stream.write(`${JSON.stringify(value)}\n`);
 }
+/** Format milliseconds as a human-readable duration string. */
 function formatDuration(ms) {
 	if (ms < 1e3) return `${ms}ms`;
 	const sec = ms / 1e3;
@@ -387,20 +307,33 @@ function formatDuration(ms) {
 	if (min < 60) return `${min}m ${remSec}s`;
 	return `${Math.floor(min / 60)}h ${min % 60}m`;
 }
+/**
+* Estimate remaining time from average completed rep duration.
+*
+* Uses a simple running mean — good enough for long suites without storing
+* per-rep history. Returns `undefined` at start and when all reps are done.
+*/
 function formatEta(totalDurationMs, completed, total) {
 	if (completed === 0 || completed >= total) return void 0;
 	const avg = totalDurationMs / completed;
 	const remaining = (total - completed) * avg;
 	return `~${formatDuration(Math.round(remaining))} remaining`;
 }
+/** Truncate error text for single-line progress rows (Unicode ellipsis). */
 function truncate(text, max) {
 	if (text.length <= max) return text;
 	return `${text.slice(0, max - 1)}…`;
 }
+/**
+* Compact per-assertion pass/fail summary for `--progress verbose` rep lines.
+*
+* @returns Comma-separated `✓ description` / `✗ description` fragments, or empty string.
+*/
 function formatAssertionSummary(results, color = false) {
 	if (!results || results.length === 0) return "";
 	return results.map((r) => `${r.passed ? okMark(color) : failMark(color)} ${r.description}`).join(", ");
 }
+/** One-line summary when a matrix cell finishes (used in default progress mode). */
 function formatCellSummary(cell, color) {
 	const mark = cell.passed ? okMark(color) : failMark(color);
 	const status = cell.passed ? passLabel(color) : failLabel(color);
@@ -417,19 +350,26 @@ function formatCellSummary(cell, color) {
 /**
 * `harness-eval grade` — LLM outcome grading on a suite report.
 */
+/** Parse an optional integer CLI flag; returns undefined when absent or invalid. */
 function optionalOptionInt(options, name) {
 	const raw = getOption(options, name);
 	if (raw === void 0) return void 0;
 	const n = Number.parseInt(raw, 10);
 	return Number.isFinite(n) ? n : void 0;
 }
+/**
+* Execute `harness-eval grade`: LLM outcome grading on a suite report JSON.
+*
+* @returns 0 when all expectations pass, 1 on failure, 2 on usage/load errors or no reps graded.
+*/
 async function gradeCommand(args) {
 	const reportPath = args.positional[0];
 	if (!reportPath) {
-		console.error("usage: harness-eval grade <report.json> [--config grading.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N]");
+		console.error("usage: harness-eval grade <report.json> [--config grading.yaml] [--suite suite.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N]");
 		return 2;
 	}
 	const configPath = getOption(args.options, "config");
+	const suitePath = getOption(args.options, "suite");
 	const expectationsPath = getOption(args.options, "expectations");
 	const outputPath = getOption(args.options, "output");
 	const model = getOption(args.options, "model");
@@ -440,8 +380,13 @@ async function gradeCommand(args) {
 	const progressMode = resolveProgressMode(args.options);
 	const useProgressColor = progressMode !== "json" && resolveProgressColor(args.options);
 	let fileConfig;
-	if (configPath) try {
-		fileConfig = await loadGradingConfig(configPath);
+	const gradingConfigPath = configPath ?? suitePath;
+	if (configPath && suitePath) {
+		console.error("grade: use only one of --config or --suite");
+		return 2;
+	}
+	if (gradingConfigPath) try {
+		fileConfig = await loadGradingConfig(gradingConfigPath);
 	} catch (err) {
 		console.error(err instanceof Error ? err.message : String(err));
 		return 2;
@@ -462,7 +407,7 @@ async function gradeCommand(args) {
 			binary,
 			timeoutMs,
 			maxConcurrent
-		}, configPath);
+		}, configPath ?? suitePath);
 	} catch (err) {
 		console.error(err instanceof Error ? err.message : String(err));
 		return 2;
@@ -489,10 +434,125 @@ async function gradeCommand(args) {
 	return gradingReportPassed(grading) ? 0 : 1;
 }
 //#endregion
+//#region src/cli/commands/pipeline.ts
+/**
+* `harness-eval pipeline` — orchestrate run → grade → envelope from suite.yaml.
+*/
+/** Read package version for envelope provenance (best-effort). */
+async function readFrameworkVersion() {
+	try {
+		const text = await readFile(join(dirname(fileURLToPath(import.meta.url)), "../../../package.json"), "utf8");
+		return JSON.parse(text).version;
+	} catch {
+		return;
+	}
+}
+/** Resolve CLI path overrides relative to the suite directory unless absolute or `~/`. */
+function resolveOverridePath(value, suiteDir) {
+	if (!value) return void 0;
+	return isAbsolute(value) || value.startsWith("~/") ? value : join(suiteDir, value);
+}
+/**
+* Execute `harness-eval pipeline`.
+*
+* @returns Step exit code (0 pass, 1 eval fail, 2 usage/load error).
+*/
+async function pipelineCommand(args) {
+	const suitePath = args.positional[0];
+	if (!suitePath) {
+		console.error("usage: harness-eval pipeline <suite.yaml|dir> [--steps run,grade,envelope] [--output path] [--grading path] [--report path] [--max-concurrent N] [--progress default|quiet|verbose|json]");
+		return 2;
+	}
+	let doc;
+	try {
+		doc = await loadSuiteDocument(suitePath);
+	} catch (err) {
+		console.error(err instanceof Error ? err.message : String(err));
+		return 2;
+	}
+	if (!doc.pipeline) {
+		console.error("suite.yaml has no pipeline block; use run, grade, and envelope commands separately");
+		return 2;
+	}
+	const suiteDir = suiteDirectoryFromPath(doc.suitePath);
+	const steps = getOption(args.options, "steps");
+	const maxConcurrent = getOptionInt(args.options, "max-concurrent", 4);
+	const progressMode = resolveProgressMode(args.options);
+	const useProgressColor = progressMode !== "json" && resolveProgressColor(args.options);
+	const projection = parseEnvelopeProjection(getOption(args.options, "projection"));
+	if (getOption(args.options, "projection") && !projection) {
+		console.error("invalid --projection; expected envelope, trajectory, or instances");
+		return 2;
+	}
+	const overrides = {};
+	const runOutput = getOption(args.options, "output");
+	if (runOutput) overrides.run = {
+		output: resolveOverridePath(runOutput, suiteDir),
+		maxConcurrent
+	};
+	const reportOverride = getOption(args.options, "report");
+	if (reportOverride) {
+		overrides.grade = {
+			...overrides.grade,
+			input: resolveOverridePath(reportOverride, suiteDir)
+		};
+		overrides.envelope = {
+			...overrides.envelope,
+			report: resolveOverridePath(reportOverride, suiteDir)
+		};
+	}
+	const gradingOutput = getOption(args.options, "grading-output");
+	if (gradingOutput) overrides.grade = {
+		...overrides.grade,
+		output: resolveOverridePath(gradingOutput, suiteDir)
+	};
+	const gradingInput = getOption(args.options, "grading");
+	if (gradingInput) overrides.envelope = {
+		...overrides.envelope,
+		grading: resolveOverridePath(gradingInput, suiteDir)
+	};
+	const envelopeOutput = getOption(args.options, "envelope-output");
+	if (envelopeOutput) overrides.envelope = {
+		...overrides.envelope,
+		output: resolveOverridePath(envelopeOutput, suiteDir)
+	};
+	if (projection) overrides.envelope = {
+		...overrides.envelope,
+		projection
+	};
+	if (doc.pipeline.grade && !doc.judge) {
+		console.error("pipeline grade step requires inline judge: block in suite.yaml");
+		return 2;
+	}
+	const frameworkVersion = await readFrameworkVersion();
+	try {
+		return (await runPipeline(doc, {
+			steps,
+			maxConcurrent,
+			overrides,
+			frameworkVersion,
+			onRunProgress: createRunProgressHandler({
+				mode: progressMode,
+				maxConcurrent,
+				color: useProgressColor
+			}),
+			onGradeProgress: createGradeProgressHandler({
+				mode: progressMode,
+				maxConcurrent,
+				color: useProgressColor
+			})
+		})).exitCode;
+	} catch (err) {
+		console.error(err instanceof Error ? err.message : String(err));
+		return 2;
+	}
+}
+//#endregion
 //#region src/cli/commands/otel-output.ts
 /**
 * Write OTLP JSON artifacts from a suite report.
 */
+/** Sanitize case/cell labels for use in OTLP artifact filenames. */
 function safeFilePart(value) {
 	return value.replace(/[^a-zA-Z0-9._-]+/g, "_");
 }
@@ -521,6 +581,11 @@ async function writeOtelArtifacts(suite, report, outputDir) {
 /**
 * `harness-eval run` command.
 */
+/**
+* Execute `harness-eval run`: load suite, run repetitions, format report.
+*
+* @returns 0 when all cells pass thresholds, 1 on assertion failure, 2 on usage/load errors.
+*/
 async function runCommand(args) {
 	const suitePath = args.positional[0];
 	if (!suitePath) {
@@ -575,14 +640,17 @@ async function runCommand(args) {
 //#endregion
 //#region src/cli/main.ts
 /**
-* CLI entry point.
+* CLI entry point — dispatches subcommands and prints usage on `--help`.
+*
+* Exit codes: 0 success, 1 eval/grade failure, 2 usage or load errors.
 */
 const USAGE = `harness-eval — harness-level eval framework
 
 Usage:
   harness-eval run <suite.yaml> [--max-concurrent N] [--baseline path] [--output path] [--otel-output dir] [--format console|markdown|json] [--adapter id] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
-  harness-eval grade <report.json> [--config grading.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N] [--format console|json] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
-  harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances|agent-trace] [--include-raw-stream-events] [--no-transcript]
+  harness-eval grade <report.json> [--config grading.yaml] [--suite suite.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N] [--format console|json] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
+  harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]
+  harness-eval pipeline <suite.yaml|dir> [--steps run,grade,envelope] [--output path] [--grading path] [--grading-output path] [--envelope-output path] [--report path] [--projection envelope|trajectory|instances] [--max-concurrent N] [--progress default|quiet|verbose|json]
   harness-eval format <report.json> [--format console|markdown|json] [--baseline path]
   harness-eval --help
 
@@ -594,6 +662,11 @@ Usage:
   --no-color        disable ANSI colors on progress output
   --color           force ANSI colors on progress output
 `;
+/**
+* Route argv to the appropriate subcommand handler.
+*
+* @returns Process exit code (0 = success, 1 = eval failure, 2 = usage error).
+*/
 async function main(argv) {
 	const parsed = parseArgs(argv);
 	if (parsed.options.help || parsed.command === "help" || parsed.options.h) {
@@ -604,6 +677,7 @@ async function main(argv) {
 		case "run": return await runCommand(parsed);
 		case "grade": return await gradeCommand(parsed);
 		case "envelope": return await envelopeCommand(parsed);
+		case "pipeline": return await pipelineCommand(parsed);
 		case "format": return await formatCommand(parsed);
 		case void 0:
 			console.error(USAGE);
@@ -615,6 +689,9 @@ async function main(argv) {
 }
 //#endregion
 //#region src/cli/bin.ts
+/**
+* CLI executable entry point — delegates to {@link main} and exits with its code.
+*/
 const code = await main(process.argv.slice(2));
 process.exit(code);
 //#endregion