@alis-build/harness-eval 0.1.1 → 0.1.3

package/dist/types-Bac8_Ixb.js

@@ -0,0 +1,246 @@
+//#region src/types/stream.ts
+/** Type guards. Prefer these over manual `e.type === "..."` checks at call sites. */
+function isSystemInit(e) {
+	return e.type === "system" && e.subtype === "init";
+}
+function isSystemRetry(e) {
+	return e.type === "system" && e.subtype === "api_retry";
+}
+function isAssistantMessage(e) {
+	return e.type === "assistant";
+}
+function isUserMessage(e) {
+	return e.type === "user";
+}
+function isResult(e) {
+	return e.type === "result";
+}
+function isTextBlock(b) {
+	return b.type === "text";
+}
+function isToolUseBlock(b) {
+	return b.type === "tool_use";
+}
+function isToolResultBlock(b) {
+	return b.type === "tool_result";
+}
+//#endregion
+//#region src/types/trajectory.ts
+/**
+* Extract the MCP namespace prefix from a tool name.
+*
+* Claude Code formats MCP tool names as `mcp__<server>__<tool>`. The namespace
+* is the first two segments joined: `mcp__<server>`. Returns null for non-MCP
+* tool names (built-ins like `Bash`, `Read`, `Edit`).
+*
+* @example
+*   namespaceOf("mcp__api__search_skills") // "mcp__api"
+*   namespaceOf("Bash")                     // null
+*/
+function namespaceOf(toolName) {
+	if (!toolName.startsWith("mcp__")) return null;
+	const parts = toolName.split("__");
+	if (parts.length < 3) return null;
+	return `${parts[0]}__${parts[1]}`;
+}
+//#endregion
+//#region src/trajectory/builder.ts
+/**
+* TrajectoryBuilder — consumes a stream of {@link StreamEvent} values and
+* produces a {@link TrajectoryView}.
+*
+* State machine: the builder is a small, tolerant state machine. Invariants:
+*
+*   - Exactly one `system/init` event opens the session. The builder requires
+*     it to be present before `build()`.
+*   - Each `assistant` event begins a new turn. Text blocks accumulate into
+*     the turn's text; `tool_use` blocks become `ToolCall` records.
+*   - `user` events with `tool_result` blocks deliver tool results back. We
+*     match them to pending calls by `tool_use_id`.
+*   - One `result` event closes the session and carries aggregate usage.
+*
+* The builder is *tolerant of partial streams*: a process killed mid-run
+* produces a coherent (but flagged) view. Tool calls without matching results
+* keep `result: null`. The `success` flag reflects whether a successful result
+* event was actually observed.
+*
+* Why a class (not a reducer)?
+*   The internal `pendingCalls` map is mutable by design — we modify ToolCall
+*   objects in place when results arrive, so other parts of the view (which
+*   hold references to the same objects) see the update for free. A reducer
+*   would force a deep copy per result event, which is wasteful and would
+*   complicate identity-based queries.
+*/
+var TrajectoryBuilder = class {
+	meta = null;
+	sessionStartTs = null;
+	turns = [];
+	allToolCalls = [];
+	/**
+	* tool_use_id → ToolCall, for matching results back to calls.
+	* Entries are removed once a result is observed.
+	*/
+	pendingCalls = /* @__PURE__ */ new Map();
+	retries = [];
+	finalUsage = null;
+	finalCostUsd = 0;
+	finalDurationMs = 0;
+	finalNumTurns = 0;
+	finalResultText = "";
+	sawResultEvent = false;
+	resultIsError = false;
+	/**
+	* Consume one event. Safe to call with events in stream order.
+	*
+	* Unknown event types are silently ignored — the schema evolves and we
+	* don't want CI to break on a new event type we haven't modelled.
+	*/
+	consume(event) {
+		if (isSystemInit(event)) {
+			this.meta = {
+				sessionId: event.session_id,
+				model: event.model,
+				cwd: event.cwd,
+				permissionMode: event.permissionMode,
+				availableTools: event.tools ?? [],
+				mcpServers: (event.mcp_servers ?? []).map((s) => ({
+					name: s.name,
+					status: s.status
+				}))
+			};
+			this.sessionStartTs = Date.now();
+			return;
+		}
+		if (event.type === "system" && event.subtype === "api_retry") {
+			this.retries.push({
+				offsetMs: this.sessionStartTs ? Date.now() - this.sessionStartTs : 0,
+				raw: event
+			});
+			return;
+		}
+		if (isAssistantMessage(event)) {
+			this.handleAssistantMessage(event);
+			return;
+		}
+		if (isUserMessage(event)) {
+			this.handleUserMessage(event);
+			return;
+		}
+		if (isResult(event)) {
+			this.sawResultEvent = true;
+			this.resultIsError = event.is_error;
+			this.finalUsage = event.usage ?? null;
+			this.finalCostUsd = event.total_cost_usd ?? 0;
+			this.finalDurationMs = event.duration_ms ?? 0;
+			this.finalNumTurns = event.num_turns ?? 0;
+			this.finalResultText = event.result ?? "";
+			return;
+		}
+	}
+	/**
+	* Finalize the view. Call after consuming the last event from the stream.
+	*
+	* Throws if no `system/init` was observed — at that point we have no model,
+	* no session id, and no available-tools list, which means assertions like
+	* "called any mcp__api__* tool" can't even be evaluated meaningfully.
+	*/
+	build() {
+		if (this.meta === null) throw new Error("TrajectoryBuilder.build() called before any system/init event was observed. The harness may have failed to start, or the stream was truncated before init.");
+		const lastTurn = this.turns[this.turns.length - 1];
+		const accumulatedText = this.turns.map((t) => t.text).filter((t) => t.length > 0).join("\n\n").trim();
+		return {
+			meta: this.meta,
+			toolCalls: this.allToolCalls,
+			turns: this.turns,
+			finalResponse: accumulatedText || this.finalResultText,
+			finalStopReason: lastTurn?.stopReason ?? null,
+			usage: {
+				inputTokens: this.finalUsage?.input_tokens ?? 0,
+				outputTokens: this.finalUsage?.output_tokens ?? 0,
+				totalCostUsd: this.finalCostUsd,
+				durationMs: this.finalDurationMs,
+				numTurns: this.finalNumTurns || this.turns.length
+			},
+			retries: this.retries,
+			success: this.sawResultEvent && !this.resultIsError
+		};
+	}
+	handleAssistantMessage(event) {
+		const turnIndex = this.turns.length;
+		const textChunks = [];
+		const toolCallsThisTurn = [];
+		for (const block of event.message.content) {
+			if (isTextBlock(block)) {
+				textChunks.push(block.text);
+				continue;
+			}
+			if (isToolUseBlock(block)) {
+				const call = {
+					name: block.name,
+					namespace: namespaceOf(block.name),
+					callId: block.id,
+					args: block.input,
+					result: null,
+					isError: false,
+					turnIndex,
+					callIndex: this.allToolCalls.length
+				};
+				this.allToolCalls.push(call);
+				this.pendingCalls.set(block.id, call);
+				toolCallsThisTurn.push(call);
+				continue;
+			}
+		}
+		this.turns.push({
+			turnIndex,
+			text: textChunks.join("").trim(),
+			toolCalls: toolCallsThisTurn,
+			stopReason: event.message.stop_reason ?? null
+		});
+	}
+	handleUserMessage(event) {
+		const content = event.message.content;
+		if (typeof content === "string") return;
+		for (const block of content) {
+			if (!isToolResultBlock(block)) continue;
+			const call = this.pendingCalls.get(block.tool_use_id);
+			if (!call) continue;
+			call.result = block.content;
+			call.isError = block.is_error ?? false;
+			this.pendingCalls.delete(block.tool_use_id);
+		}
+	}
+};
+/**
+* Convenience: drain an async iterable of events through a fresh builder.
+*
+* Suitable when you have the full event stream and just want the view.
+* For interactive/incremental scenarios (e.g. surfacing partial state in a UI)
+* instantiate {@link TrajectoryBuilder} directly and call `consume()` /
+* `build()` yourself.
+*/
+async function buildTrajectory(events) {
+	const builder = new TrajectoryBuilder();
+	for await (const event of events) builder.consume(event);
+	return builder.build();
+}
+//#endregion
+//#region src/adapters/types.ts
+/**
+* Thrown when the harness fails to produce a usable trajectory.
+*
+* Most commonly this means the process failed before emitting a usable
+* session init event. Inspect `diagnostics.stderr` for the cause.
+*/
+var AdapterError = class extends Error {
+	diagnostics;
+	constructor(message, diagnostics) {
+		super(message);
+		this.diagnostics = diagnostics;
+		this.name = "AdapterError";
+	}
+};
+//#endregion
+export { isAssistantMessage as a, isSystemRetry as c, isToolUseBlock as d, isUserMessage as f, namespaceOf as i, isTextBlock as l, TrajectoryBuilder as n, isResult as o, buildTrajectory as r, isSystemInit as s, AdapterError as t, isToolResultBlock as u };
+
+//# sourceMappingURL=types-Bac8_Ixb.js.map

package/dist/types-Bac8_Ixb.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types-Bac8_Ixb.js","names":[],"sources":["../src/types/stream.ts","../src/types/trajectory.ts","../src/trajectory/builder.ts","../src/adapters/types.ts"],"sourcesContent":["/**\n * Discriminated union of events emitted by Claude Code's\n * `--output-format stream-json` mode.\n *\n * The format is NDJSON (one JSON object per line on stdout). Each line has\n * a required `type` field and often a `subtype` for further disambiguation.\n *\n * Source notes: the stream-json schema is not formally documented as of mid-2026.\n * These types are derived from:\n *   - https://code.claude.com/docs/en/headless\n *   - https://github.com/anthropics/claude-code/issues/24612 (event-types tracking issue)\n *   - https://takopi.dev/reference/runners/claude/stream-json-cheatsheet/\n *   - The `@anthropic-ai/claude-agent-sdk` TypeScript declaration files,\n *     which are the de-facto source of truth.\n *\n * When adding new event types, prefer extending the union here rather than\n * branching on `any` in callers. Unknown events should be tolerated silently\n * by the builder (the schema evolves and we don't want CI to break on a new\n * event type we haven't modelled yet).\n */\n\n/** Top-level discriminated union of stream-json events. */\nexport type StreamEvent =\n  | SystemInitEvent\n  | SystemRetryEvent\n  | SystemPluginInstallEvent\n  | SystemCompactBoundaryEvent\n  | SystemUnknownEvent\n  | AssistantMessageEvent\n  | UserMessageEvent\n  | ResultEvent;\n\n// system events\n\n/** Emitted once at session start. Carries the session-level metadata. */\nexport interface SystemInitEvent {\n  type: \"system\";\n  subtype: \"init\";\n  session_id: string;\n  cwd: string;\n  model: string;\n  permissionMode?: string;\n  apiKeySource?: string;\n  /** Names of tools available in the session (built-in + MCP). */\n  tools: string[];\n  /** MCP servers configured for this session, with connection status. */\n  mcp_servers: McpServerStatus[];\n}\n\nexport interface McpServerStatus {\n  name: string;\n  status: \"connected\" | \"disconnected\" | \"error\" | string;\n}\n\n/** Emitted when the API rate-limits us or otherwise asks for a retry. */\nexport interface SystemRetryEvent {\n  type: \"system\";\n  subtype: \"api_retry\";\n  session_id: string;\n  /** Implementation-defined retry payload (delay, reason, etc). */\n  [key: string]: unknown;\n}\n\n/** Emitted while marketplace plugins are installing pre-session. */\nexport interface SystemPluginInstallEvent {\n  type: \"system\";\n  subtype: \"plugin_install\";\n  session_id: string;\n  [key: string]: unknown;\n}\n\n/** Emitted when Claude Code compacts the context window mid-session. */\nexport interface SystemCompactBoundaryEvent {\n  type: \"system\";\n  subtype: \"compact_boundary\";\n  session_id: string;\n  [key: string]: unknown;\n}\n\n/**\n * Catch-all for `type: \"system\"` events we haven't modelled.\n *\n * Keeps the union exhaustive while tolerating schema evolution. Callers should\n * either explicitly handle a known subtype or fall through to ignore.\n */\nexport interface SystemUnknownEvent {\n  type: \"system\";\n  subtype: string;\n  session_id?: string;\n  [key: string]: unknown;\n}\n\n// conversational events\n\n/** One assistant turn. The `message` field mirrors the Anthropic Messages API shape. */\nexport interface AssistantMessageEvent {\n  type: \"assistant\";\n  session_id: string;\n  message: AssistantMessage;\n}\n\nexport interface AssistantMessage {\n  id: string;\n  type: \"message\";\n  role: \"assistant\";\n  content: ContentBlock[];\n  model?: string;\n  stop_reason?: StopReason | null;\n  usage?: Usage;\n}\n\n/**\n * A user-role message in the stream.\n *\n * In stream-json these are usually *synthetic* — the harness injects them to\n * feed tool results back into the conversation after dispatching a tool. The\n * very first user message (the prompt) is also emitted here for completeness.\n */\nexport interface UserMessageEvent {\n  type: \"user\";\n  session_id: string;\n  message: UserMessage;\n}\n\nexport interface UserMessage {\n  role: \"user\";\n  /** String for the initial prompt, array of blocks when carrying tool results. */\n  content: ContentBlock[] | string;\n}\n\n// content blocks\n\nexport type ContentBlock = TextBlock | ToolUseBlock | ToolResultBlock;\n\nexport interface TextBlock {\n  type: \"text\";\n  text: string;\n}\n\nexport interface ToolUseBlock {\n  type: \"tool_use\";\n  /** Unique id assigned by the model; used to match tool_result back to this call. */\n  id: string;\n  /** Tool name. MCP tools follow the convention `mcp__<server>__<tool>`. */\n  name: string;\n  /** Arguments the model passed. Schema is per-tool. */\n  input: unknown;\n}\n\nexport interface ToolResultBlock {\n  type: \"tool_result\";\n  /** The id of the corresponding tool_use block. */\n  tool_use_id: string;\n  /** Tool output. May be plain text or further content blocks for richer tools. */\n  content: string | ContentBlock[];\n  is_error?: boolean;\n}\n\n// result envelope\n\n/** Emitted once at session end. Carries aggregate usage and cost. */\nexport interface ResultEvent {\n  type: \"result\";\n  subtype: \"success\" | \"error\";\n  session_id: string;\n  total_cost_usd: number;\n  is_error: boolean;\n  duration_ms: number;\n  duration_api_ms?: number;\n  num_turns: number;\n  /** The final text the harness returned, if any. */\n  result?: string;\n  usage?: Usage;\n}\n\n// shared scalars\n\n/**\n * Reasons the model can stop a turn. Open-ended string union because new\n * stop reasons appear over time.\n */\nexport type StopReason =\n  | \"end_turn\"\n  | \"tool_use\"\n  | \"max_tokens\"\n  | \"stop_sequence\"\n  | (string & {});\n\nexport interface Usage {\n  input_tokens: number;\n  output_tokens: number;\n  cache_creation_input_tokens?: number;\n  cache_read_input_tokens?: number;\n}\n\n// type guards\n\n/** Type guards. Prefer these over manual `e.type === \"...\"` checks at call sites. */\n\nexport function isSystemInit(e: StreamEvent): e is SystemInitEvent {\n  return e.type === \"system\" && (e as SystemInitEvent).subtype === \"init\";\n}\n\nexport function isSystemRetry(e: StreamEvent): e is SystemRetryEvent {\n  return e.type === \"system\" && (e as SystemRetryEvent).subtype === \"api_retry\";\n}\n\nexport function isAssistantMessage(e: StreamEvent): e is AssistantMessageEvent {\n  return e.type === \"assistant\";\n}\n\nexport function isUserMessage(e: StreamEvent): e is UserMessageEvent {\n  return e.type === \"user\";\n}\n\nexport function isResult(e: StreamEvent): e is ResultEvent {\n  return e.type === \"result\";\n}\n\nexport function isTextBlock(b: ContentBlock): b is TextBlock {\n  return b.type === \"text\";\n}\n\nexport function isToolUseBlock(b: ContentBlock): b is ToolUseBlock {\n  return b.type === \"tool_use\";\n}\n\nexport function isToolResultBlock(b: ContentBlock): b is ToolResultBlock {\n  return b.type === \"tool_result\";\n}\n","/**\n * TrajectoryView — the assertion-friendly projection of a Claude Code session.\n *\n * The view is derived from the stream of {@link StreamEvent} values produced by\n * the harness, but is optimized for the queries that the assertion DSL needs to\n * express:\n *\n *   - did tool X get called?  (look at `toolCalls`)\n *   - did tool A come before tool B?  (compare `turnIndex` / `callIndex`)\n *   - was a tool called with arguments matching predicate P?  (`toolCalls[i].args`)\n *   - did the agent answer without using any tool?  (`toolCalls.length === 0`)\n *\n * The view is reconstructable from the raw events (lossless w.r.t. assertions),\n * but operating on it directly is dramatically simpler than walking event\n * streams or OTel span trees.\n *\n * Design notes:\n *  - `turnIndex` and `callIndex` are the right primitives for ordering.\n *    Wall-clock timestamps from the stream are unreliable for sub-second\n *    ordering and parallel tool dispatch.\n *  - Parallel tool calls (multiple `tool_use` blocks in one assistant message)\n *    share a `turnIndex` but have distinct `callIndex` values in emission order.\n *  - `namespace` is precomputed so assertions like `called(pattern: \"mcp__api__*\")`\n *    can do a cheap string check.\n */\n\nimport type { StopReason } from \"./stream\";\n\nexport interface TrajectoryView {\n  /** Session metadata, captured from the `system/init` event. */\n  meta: SessionMeta;\n\n  /** Every tool call, in global emission order. */\n  toolCalls: ToolCall[];\n\n  /** Each assistant turn: text content + any tool calls emitted in that turn. */\n  turns: AssistantTurn[];\n\n  /** All assistant text concatenated across turns. Useful for `response_contains`. */\n  finalResponse: string;\n\n  /** Stop reason of the *last* assistant turn. */\n  finalStopReason: StopReason | null;\n\n  /** Aggregate usage and cost from the result event. */\n  usage: UsageSummary;\n\n  /** Retry events observed during the run (rate limits, transient errors). */\n  retries: RetryRecord[];\n\n  /** Whether the result envelope indicated success. */\n  success: boolean;\n}\n\nexport interface SessionMeta {\n  sessionId: string;\n  model: string;\n  cwd: string;\n  permissionMode?: string;\n  /** Tool names the harness reported as available at session start. */\n  availableTools: string[];\n  /** MCP servers configured for the session, with connection status. */\n  mcpServers: { name: string; status: string }[];\n}\n\nexport interface ToolCall {\n  /** Fully-qualified tool name, e.g. `\"mcp__api__search_skills\"` or `\"Bash\"`. */\n  name: string;\n\n  /**\n   * Namespace prefix for MCP-style names (`\"mcp__api\"`), or null for built-ins.\n   * Precomputed via {@link namespaceOf} for cheap pattern matching.\n   */\n  namespace: string | null;\n\n  /** The `tool_use` block's `id`; matches a later `tool_result.tool_use_id`. */\n  callId: string;\n\n  /** Args the model emitted on this call. Tool-specific schema. */\n  args: unknown;\n\n  /** Tool result, or null if no result was observed (e.g. process killed). */\n  result: unknown | null;\n\n  /** Whether the tool reported an error in its result. */\n  isError: boolean;\n\n  /**\n   * Which assistant turn produced this call. Parallel calls within a single\n   * assistant message share a `turnIndex`.\n   */\n  turnIndex: number;\n\n  /** Index in the global ordered tool-call sequence. */\n  callIndex: number;\n}\n\nexport interface AssistantTurn {\n  turnIndex: number;\n  /** Text emitted in this turn (may be empty if turn was tool-only). */\n  text: string;\n  /** Tool calls emitted in this turn, in their block order. */\n  toolCalls: ToolCall[];\n  /** Stop reason reported by the model for this turn. */\n  stopReason: StopReason | null;\n}\n\nexport interface UsageSummary {\n  inputTokens: number;\n  outputTokens: number;\n  totalCostUsd: number;\n  durationMs: number;\n  numTurns: number;\n}\n\nexport interface RetryRecord {\n  /** ms since session start (approximate; the stream doesn't include precise ts). */\n  offsetMs: number;\n  /** Raw payload from the `system/api_retry` event for diagnostics. */\n  raw: unknown;\n}\n\n// helpers\n\n/**\n * Extract the MCP namespace prefix from a tool name.\n *\n * Claude Code formats MCP tool names as `mcp__<server>__<tool>`. The namespace\n * is the first two segments joined: `mcp__<server>`. Returns null for non-MCP\n * tool names (built-ins like `Bash`, `Read`, `Edit`).\n *\n * @example\n *   namespaceOf(\"mcp__api__search_skills\") // \"mcp__api\"\n *   namespaceOf(\"Bash\")                     // null\n */\nexport function namespaceOf(toolName: string): string | null {\n  if (!toolName.startsWith(\"mcp__\")) return null;\n  const parts = toolName.split(\"__\");\n  if (parts.length < 3) return null;\n  return `${parts[0]}__${parts[1]}`;\n}\n","/**\n * TrajectoryBuilder — consumes a stream of {@link StreamEvent} values and\n * produces a {@link TrajectoryView}.\n *\n * State machine: the builder is a small, tolerant state machine. Invariants:\n *\n *   - Exactly one `system/init` event opens the session. The builder requires\n *     it to be present before `build()`.\n *   - Each `assistant` event begins a new turn. Text blocks accumulate into\n *     the turn's text; `tool_use` blocks become `ToolCall` records.\n *   - `user` events with `tool_result` blocks deliver tool results back. We\n *     match them to pending calls by `tool_use_id`.\n *   - One `result` event closes the session and carries aggregate usage.\n *\n * The builder is *tolerant of partial streams*: a process killed mid-run\n * produces a coherent (but flagged) view. Tool calls without matching results\n * keep `result: null`. The `success` flag reflects whether a successful result\n * event was actually observed.\n *\n * Why a class (not a reducer)?\n *   The internal `pendingCalls` map is mutable by design — we modify ToolCall\n *   objects in place when results arrive, so other parts of the view (which\n *   hold references to the same objects) see the update for free. A reducer\n *   would force a deep copy per result event, which is wasteful and would\n *   complicate identity-based queries.\n */\n\nimport {\n  isAssistantMessage,\n  isResult,\n  isSystemInit,\n  isTextBlock,\n  isToolResultBlock,\n  isToolUseBlock,\n  isUserMessage,\n  type StreamEvent,\n  type Usage,\n} from \"../types/stream\";\nimport {\n  namespaceOf,\n  type AssistantTurn,\n  type RetryRecord,\n  type SessionMeta,\n  type ToolCall,\n  type TrajectoryView,\n} from \"../types/trajectory\";\n\nexport class TrajectoryBuilder {\n  private meta: SessionMeta | null = null;\n  private sessionStartTs: number | null = null;\n\n  private turns: AssistantTurn[] = [];\n  private allToolCalls: ToolCall[] = [];\n\n  /**\n   * tool_use_id → ToolCall, for matching results back to calls.\n   * Entries are removed once a result is observed.\n   */\n  private pendingCalls: Map<string, ToolCall> = new Map();\n\n  private retries: RetryRecord[] = [];\n\n  private finalUsage: Usage | null = null;\n  private finalCostUsd = 0;\n  private finalDurationMs = 0;\n  private finalNumTurns = 0;\n  private finalResultText = \"\";\n  private sawResultEvent = false;\n  private resultIsError = false;\n\n  /**\n   * Consume one event. Safe to call with events in stream order.\n   *\n   * Unknown event types are silently ignored — the schema evolves and we\n   * don't want CI to break on a new event type we haven't modelled.\n   */\n  consume(event: StreamEvent): void {\n    if (isSystemInit(event)) {\n      this.meta = {\n        sessionId: event.session_id,\n        model: event.model,\n        cwd: event.cwd,\n        permissionMode: event.permissionMode,\n        availableTools: event.tools ?? [],\n        mcpServers: (event.mcp_servers ?? []).map((s) => ({\n          name: s.name,\n          status: s.status,\n        })),\n      };\n      this.sessionStartTs = Date.now();\n      return;\n    }\n\n    if (event.type === \"system\" && event.subtype === \"api_retry\") {\n      this.retries.push({\n        offsetMs: this.sessionStartTs ? Date.now() - this.sessionStartTs : 0,\n        raw: event,\n      });\n      return;\n    }\n\n    if (isAssistantMessage(event)) {\n      this.handleAssistantMessage(event);\n      return;\n    }\n\n    if (isUserMessage(event)) {\n      this.handleUserMessage(event);\n      return;\n    }\n\n    if (isResult(event)) {\n      this.sawResultEvent = true;\n      this.resultIsError = event.is_error;\n      this.finalUsage = event.usage ?? null;\n      this.finalCostUsd = event.total_cost_usd ?? 0;\n      this.finalDurationMs = event.duration_ms ?? 0;\n      this.finalNumTurns = event.num_turns ?? 0;\n      this.finalResultText = event.result ?? \"\";\n      return;\n    }\n\n    // Unknown event: ignored. See class doc.\n  }\n\n  /**\n   * Finalize the view. Call after consuming the last event from the stream.\n   *\n   * Throws if no `system/init` was observed — at that point we have no model,\n   * no session id, and no available-tools list, which means assertions like\n   * \"called any mcp__api__* tool\" can't even be evaluated meaningfully.\n   */\n  build(): TrajectoryView {\n    if (this.meta === null) {\n      throw new Error(\n        \"TrajectoryBuilder.build() called before any system/init event was observed. \" +\n          \"The harness may have failed to start, or the stream was truncated before init.\",\n      );\n    }\n\n    const lastTurn = this.turns[this.turns.length - 1];\n\n    // Prefer the assistant text we accumulated turn-by-turn over the\n    // `result.result` field, because the latter is sometimes a summary\n    // and the former is exactly what the model said.\n    const accumulatedText = this.turns\n      .map((t) => t.text)\n      .filter((t) => t.length > 0)\n      .join(\"\\n\\n\")\n      .trim();\n\n    return {\n      meta: this.meta,\n      toolCalls: this.allToolCalls,\n      turns: this.turns,\n      finalResponse: accumulatedText || this.finalResultText,\n      finalStopReason: lastTurn?.stopReason ?? null,\n      usage: {\n        inputTokens: this.finalUsage?.input_tokens ?? 0,\n        outputTokens: this.finalUsage?.output_tokens ?? 0,\n        totalCostUsd: this.finalCostUsd,\n        durationMs: this.finalDurationMs,\n        // Fall back to observed turn count if the result event was missing.\n        numTurns: this.finalNumTurns || this.turns.length,\n      },\n      retries: this.retries,\n      // Successful = saw a non-error result envelope. Streams that ended without\n      // a result event are reported as unsuccessful regardless of tool outcomes.\n      success: this.sawResultEvent && !this.resultIsError,\n    };\n  }\n\n  // private handlers\n\n  private handleAssistantMessage(\n    event: Extract<StreamEvent, { type: \"assistant\" }>,\n  ): void {\n    const turnIndex = this.turns.length;\n    const textChunks: string[] = [];\n    const toolCallsThisTurn: ToolCall[] = [];\n\n    for (const block of event.message.content) {\n      if (isTextBlock(block)) {\n        textChunks.push(block.text);\n        continue;\n      }\n      if (isToolUseBlock(block)) {\n        const call: ToolCall = {\n          name: block.name,\n          namespace: namespaceOf(block.name),\n          callId: block.id,\n          args: block.input,\n          result: null,\n          isError: false,\n          turnIndex,\n          callIndex: this.allToolCalls.length,\n        };\n        this.allToolCalls.push(call);\n        this.pendingCalls.set(block.id, call);\n        toolCallsThisTurn.push(call);\n        continue;\n      }\n      // tool_result blocks don't appear in assistant messages — those arrive\n      // via user messages. If one does appear, ignore it; we'd rather drop\n      // an unexpected block than crash the eval.\n    }\n\n    this.turns.push({\n      turnIndex,\n      text: textChunks.join(\"\").trim(),\n      toolCalls: toolCallsThisTurn,\n      stopReason: event.message.stop_reason ?? null,\n    });\n  }\n\n  private handleUserMessage(\n    event: Extract<StreamEvent, { type: \"user\" }>,\n  ): void {\n    const content = event.message.content;\n\n    // The very first user message carries the prompt as a plain string. We\n    // already know the prompt (the caller passed it to the adapter), so we\n    // ignore this case — there's nothing assertion-relevant in it.\n    if (typeof content === \"string\") return;\n\n    for (const block of content) {\n      if (!isToolResultBlock(block)) continue;\n\n      const call = this.pendingCalls.get(block.tool_use_id);\n      if (!call) {\n        // Unmatched result: ignore. Can happen if events arrive out of order\n        // or the corresponding tool_use was emitted in an earlier run that\n        // we're resuming. Either way, dropping is safer than throwing.\n        continue;\n      }\n\n      call.result = block.content;\n      call.isError = block.is_error ?? false;\n      this.pendingCalls.delete(block.tool_use_id);\n    }\n  }\n}\n\n/**\n * Convenience: drain an async iterable of events through a fresh builder.\n *\n * Suitable when you have the full event stream and just want the view.\n * For interactive/incremental scenarios (e.g. surfacing partial state in a UI)\n * instantiate {@link TrajectoryBuilder} directly and call `consume()` /\n * `build()` yourself.\n */\nexport async function buildTrajectory(\n  events: AsyncIterable<StreamEvent>,\n): Promise<TrajectoryView> {\n  const builder = new TrajectoryBuilder();\n  for await (const event of events) {\n    builder.consume(event);\n  }\n  return builder.build();\n}\n","/**\n * Generic harness adapter contract.\n *\n * Every harness adapter produces a {@link TrajectoryView} plus process\n * diagnostics. The runner and assertion engine depend only on these types —\n * not on any specific harness implementation.\n */\n\nimport type { TrajectoryView } from \"../types/trajectory\";\n\n/** Base config every adapter must accept. */\nexport interface BaseAdapterConfig {\n  prompt: string;\n  model?: string;\n  timeoutMs?: number;\n  signal?: AbortSignal;\n  env?: Record<string, string>;\n  cwd?: string;\n}\n\n/** Suite-level config: generic fields plus adapter-specific nested blocks. */\nexport type SuiteConfig = Partial<BaseAdapterConfig> & {\n  /** Claude Code adapter options (when `adapter` is `claude-code`). */\n  claudeCode?: Record<string, unknown>;\n  /** Codex CLI adapter options (when `adapter` is `codex`). */\n  codex?: Record<string, unknown>;\n};\n\n/** Generic harness adapter interface. */\nexport interface HarnessAdapter<\n  TConfig extends BaseAdapterConfig = BaseAdapterConfig,\n> {\n  readonly id: string;\n  run(config: TConfig): Promise<AdapterResult>;\n}\n\n/** Successful adapter run. */\nexport interface AdapterResult {\n  view: TrajectoryView;\n  diagnostics: AdapterDiagnostics;\n}\n\n/** Process-level diagnostics from any adapter. */\nexport interface AdapterDiagnostics {\n  exitCode: number | null;\n  signal: NodeJS.Signals | null;\n  stderr: string;\n  parseErrors: ParseErrorRecord[];\n  timedOut: boolean;\n  durationMs: number;\n}\n\nexport interface ParseErrorRecord {\n  line: string;\n  error: string;\n}\n\n/**\n * Thrown when the harness fails to produce a usable trajectory.\n *\n * Most commonly this means the process failed before emitting a usable\n * session init event. Inspect `diagnostics.stderr` for the cause.\n */\nexport class AdapterError extends Error {\n  constructor(\n    message: string,\n    public readonly diagnostics: Partial<AdapterDiagnostics>,\n  ) {\n    super(message);\n    this.name = \"AdapterError\";\n  }\n}\n"],"mappings":";;AAuMA,SAAgB,aAAa,GAAsC;CACjE,OAAO,EAAE,SAAS,YAAa,EAAsB,YAAY;AACnE;AAEA,SAAgB,cAAc,GAAuC;CACnE,OAAO,EAAE,SAAS,YAAa,EAAuB,YAAY;AACpE;AAEA,SAAgB,mBAAmB,GAA4C;CAC7E,OAAO,EAAE,SAAS;AACpB;AAEA,SAAgB,cAAc,GAAuC;CACnE,OAAO,EAAE,SAAS;AACpB;AAEA,SAAgB,SAAS,GAAkC;CACzD,OAAO,EAAE,SAAS;AACpB;AAEA,SAAgB,YAAY,GAAiC;CAC3D,OAAO,EAAE,SAAS;AACpB;AAEA,SAAgB,eAAe,GAAoC;CACjE,OAAO,EAAE,SAAS;AACpB;AAEA,SAAgB,kBAAkB,GAAuC;CACvE,OAAO,EAAE,SAAS;AACpB;;;;;;;;;;;;;;AC9FA,SAAgB,YAAY,UAAiC;CAC3D,IAAI,CAAC,SAAS,WAAW,OAAO,GAAG,OAAO;CAC1C,MAAM,QAAQ,SAAS,MAAM,IAAI;CACjC,IAAI,MAAM,SAAS,GAAG,OAAO;CAC7B,OAAO,GAAG,MAAM,GAAG,IAAI,MAAM;AAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC7FA,IAAa,oBAAb,MAA+B;CAC7B,OAAmC;CACnC,iBAAwC;CAExC,QAAiC,CAAC;CAClC,eAAmC,CAAC;;;;;CAMpC,+BAA8C,IAAI,IAAI;CAEtD,UAAiC,CAAC;CAElC,aAAmC;CACnC,eAAuB;CACvB,kBAA0B;CAC1B,gBAAwB;CACxB,kBAA0B;CAC1B,iBAAyB;CACzB,gBAAwB;;;;;;;CAQxB,QAAQ,OAA0B;EAChC,IAAI,aAAa,KAAK,GAAG;GACvB,KAAK,OAAO;IACV,WAAW,MAAM;IACjB,OAAO,MAAM;IACb,KAAK,MAAM;IACX,gBAAgB,MAAM;IACtB,gBAAgB,MAAM,SAAS,CAAC;IAChC,aAAa,MAAM,eAAe,CAAC,EAAA,CAAG,KAAK,OAAO;KAChD,MAAM,EAAE;KACR,QAAQ,EAAE;IACZ,EAAE;GACJ;GACA,KAAK,iBAAiB,KAAK,IAAI;GAC/B;EACF;EAEA,IAAI,MAAM,SAAS,YAAY,MAAM,YAAY,aAAa;GAC5D,KAAK,QAAQ,KAAK;IAChB,UAAU,KAAK,iBAAiB,KAAK,IAAI,IAAI,KAAK,iBAAiB;IACnE,KAAK;GACP,CAAC;GACD;EACF;EAEA,IAAI,mBAAmB,KAAK,GAAG;GAC7B,KAAK,uBAAuB,KAAK;GACjC;EACF;EAEA,IAAI,cAAc,KAAK,GAAG;GACxB,KAAK,kBAAkB,KAAK;GAC5B;EACF;EAEA,IAAI,SAAS,KAAK,GAAG;GACnB,KAAK,iBAAiB;GACtB,KAAK,gBAAgB,MAAM;GAC3B,KAAK,aAAa,MAAM,SAAS;GACjC,KAAK,eAAe,MAAM,kBAAkB;GAC5C,KAAK,kBAAkB,MAAM,eAAe;GAC5C,KAAK,gBAAgB,MAAM,aAAa;GACxC,KAAK,kBAAkB,MAAM,UAAU;GACvC;EACF;CAGF;;;;;;;;CASA,QAAwB;EACtB,IAAI,KAAK,SAAS,MAChB,MAAM,IAAI,MACR,4JAEF;EAGF,MAAM,WAAW,KAAK,MAAM,KAAK,MAAM,SAAS;EAKhD,MAAM,kBAAkB,KAAK,MAC1B,KAAK,MAAM,EAAE,IAAI,CAAC,CAClB,QAAQ,MAAM,EAAE,SAAS,CAAC,CAAC,CAC3B,KAAK,MAAM,CAAC,CACZ,KAAK;EAER,OAAO;GACL,MAAM,KAAK;GACX,WAAW,KAAK;GAChB,OAAO,KAAK;GACZ,eAAe,mBAAmB,KAAK;GACvC,iBAAiB,UAAU,cAAc;GACzC,OAAO;IACL,aAAa,KAAK,YAAY,gBAAgB;IAC9C,cAAc,KAAK,YAAY,iBAAiB;IAChD,cAAc,KAAK;IACnB,YAAY,KAAK;IAEjB,UAAU,KAAK,iBAAiB,KAAK,MAAM;GAC7C;GACA,SAAS,KAAK;GAGd,SAAS,KAAK,kBAAkB,CAAC,KAAK;EACxC;CACF;CAIA,uBACE,OACM;EACN,MAAM,YAAY,KAAK,MAAM;EAC7B,MAAM,aAAuB,CAAC;EAC9B,MAAM,oBAAgC,CAAC;EAEvC,KAAK,MAAM,SAAS,MAAM,QAAQ,SAAS;GACzC,IAAI,YAAY,KAAK,GAAG;IACtB,WAAW,KAAK,MAAM,IAAI;IAC1B;GACF;GACA,IAAI,eAAe,KAAK,GAAG;IACzB,MAAM,OAAiB;KACrB,MAAM,MAAM;KACZ,WAAW,YAAY,MAAM,IAAI;KACjC,QAAQ,MAAM;KACd,MAAM,MAAM;KACZ,QAAQ;KACR,SAAS;KACT;KACA,WAAW,KAAK,aAAa;IAC/B;IACA,KAAK,aAAa,KAAK,IAAI;IAC3B,KAAK,aAAa,IAAI,MAAM,IAAI,IAAI;IACpC,kBAAkB,KAAK,IAAI;IAC3B;GACF;EAIF;EAEA,KAAK,MAAM,KAAK;GACd;GACA,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,KAAK;GAC/B,WAAW;GACX,YAAY,MAAM,QAAQ,eAAe;EAC3C,CAAC;CACH;CAEA,kBACE,OACM;EACN,MAAM,UAAU,MAAM,QAAQ;EAK9B,IAAI,OAAO,YAAY,UAAU;EAEjC,KAAK,MAAM,SAAS,SAAS;GAC3B,IAAI,CAAC,kBAAkB,KAAK,GAAG;GAE/B,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,WAAW;GACpD,IAAI,CAAC,MAIH;GAGF,KAAK,SAAS,MAAM;GACpB,KAAK,UAAU,MAAM,YAAY;GACjC,KAAK,aAAa,OAAO,MAAM,WAAW;EAC5C;CACF;AACF;;;;;;;;;AAUA,eAAsB,gBACpB,QACyB;CACzB,MAAM,UAAU,IAAI,kBAAkB;CACtC,WAAW,MAAM,SAAS,QACxB,QAAQ,QAAQ,KAAK;CAEvB,OAAO,QAAQ,MAAM;AACvB;;;;;;;;;ACpMA,IAAa,eAAb,cAAkC,MAAM;CAGpB;CAFlB,YACE,SACA,aACA;EACA,MAAM,OAAO;EAFG,KAAA,cAAA;EAGhB,KAAK,OAAO;CACd;AACF"}

package/dist/types-Bu8uOZZN.d.ts

@@ -0,0 +1,77 @@
+import { i as BaseAdapterConfig, r as AdapterResult, x as StreamEvent } from "./types-C0gBkl0-.js";
+
+//#region src/adapters/codex/types.d.ts
+/** Codex sandbox policies (`codex exec --sandbox`). */
+type CodexSandbox = "read-only" | "workspace-write" | "danger-full-access";
+/** Codex approval modes (`--ask-for-approval`). */
+type CodexAskForApproval = "untrusted" | "on-request" | "never";
+/** Codex-specific options (nested under `codex` in YAML). */
+interface CodexOptions {
+  binary?: string;
+  model?: string;
+  profile?: string;
+  sandbox?: CodexSandbox;
+  addDirs?: string[];
+  /** Inline `-c key=value` overrides (repeatable on CLI). */
+  configOverrides?: string[];
+  askForApproval?: CodexAskForApproval;
+  dangerouslyBypassApprovalsAndSandbox?: boolean;
+  dangerouslyBypassHookTrust?: boolean;
+  ephemeral?: boolean;
+  ignoreUserConfig?: boolean;
+  skipGitRepoCheck?: boolean;
+  outputSchema?: string;
+  outputLastMessage?: string;
+  /**
+   * When true (default), harness runs auto-generate a temp `--output-last-message`
+   * path and read it back as `finalResponse` if JSONL has no assistant_message.
+   */
+  captureLastMessage?: boolean;
+  /**
+   * When true, each run uses a fresh temp `$CODEX_HOME` for isolation.
+   * Default false — inherit caller's ~/.codex config and auth.
+   */
+  isolateConfig?: boolean;
+}
+/** Configuration for a single Codex harness run. */
+interface CodexAdapterConfig extends BaseAdapterConfig, CodexOptions {}
+/** Codex run result — includes mapped stream events for debugging. */
+interface CodexAdapterResult extends AdapterResult {
+  rawEvents: StreamEvent[];
+}
+/** Raw Codex `--json` thread event (partial — tolerate unknown fields). */
+interface CodexJsonEvent {
+  type?: string;
+  thread_id?: string;
+  usage?: CodexUsage;
+  item?: CodexItem;
+  message?: string;
+}
+/** Token usage on a Codex turn or thread event. */
+interface CodexUsage {
+  input_tokens?: number;
+  cached_input_tokens?: number;
+  output_tokens?: number;
+  reasoning_output_tokens?: number;
+}
+/** One item in a Codex thread (tool call, command, or assistant message). */
+interface CodexItem {
+  id?: string;
+  type?: string;
+  item_type?: string;
+  server?: string;
+  tool?: string;
+  arguments?: unknown;
+  command?: string;
+  exit_code?: number;
+  aggregated_output?: string;
+  text?: string;
+  result?: unknown;
+  error?: {
+    message?: string;
+  } | null;
+  status?: string;
+}
+//#endregion
+export { CodexOptions as i, CodexAdapterResult as n, CodexJsonEvent as r, CodexAdapterConfig as t };
+//# sourceMappingURL=types-Bu8uOZZN.d.ts.map

package/dist/{types-B9H4IZtA.d.ts → types-C0gBkl0-.d.ts}

@@ -265,7 +265,8 @@ interface BaseAdapterConfig {
 }
 /** Suite-level config: generic fields plus adapter-specific nested blocks. */
 type SuiteConfig = Partial<BaseAdapterConfig> & {
-  /** Claude Code adapter options (when `adapter` is `claude-code`). */claudeCode?: Record<string, unknown>;
+  /** Claude Code adapter options (when `adapter` is `claude-code`). */claudeCode?: Record<string, unknown>; /** Codex CLI adapter options (when `adapter` is `codex`). */
+  codex?: Record<string, unknown>;
 };
 /** Generic harness adapter interface. */
 interface HarnessAdapter<TConfig extends BaseAdapterConfig = BaseAdapterConfig> {
@@ -302,4 +303,4 @@ declare class AdapterError extends Error {
 }
 //#endregion
 export { Usage as A, isUserMessage as B, SystemInitEvent as C, TextBlock as D, SystemUnknownEvent as E, isSystemInit as F, isSystemRetry as I, isTextBlock as L, UserMessageEvent as M, isAssistantMessage as N, ToolResultBlock as O, isResult as P, isToolResultBlock as R, SystemCompactBoundaryEvent as S, SystemRetryEvent as T, ContentBlock as _, HarnessAdapter as a, StopReason as b, AssistantTurn as c, ToolCall as d, TrajectoryView as f, AssistantMessageEvent as g, AssistantMessage as h, BaseAdapterConfig as i, UserMessage as j, ToolUseBlock as k, RetryRecord as l, namespaceOf as m, AdapterError as n, ParseErrorRecord as o, UsageSummary as p, AdapterResult as r, SuiteConfig as s, AdapterDiagnostics as t, SessionMeta as u, McpServerStatus as v, SystemPluginInstallEvent as w, StreamEvent as x, ResultEvent as y, isToolUseBlock as z };
-//# sourceMappingURL=types-B9H4IZtA.d.ts.map
+//# sourceMappingURL=types-C0gBkl0-.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@alis-build/harness-eval",
-    "version": "0.1.1",
+    "version": "0.1.3",
     "description": "Harness-level eval framework for measuring AI coding agent tool-selection behavior",
     "type": "module",
     "main": "./dist/index.js",
@@ -27,6 +27,10 @@
             "types": "./dist/adapters/claude-code/index.d.ts",
             "import": "./dist/adapters/claude-code/index.js"
         },
+        "./adapters/codex": {
+            "types": "./dist/adapters/codex/index.d.ts",
+            "import": "./dist/adapters/codex/index.js"
+        },
         "./runner": {
             "types": "./dist/runner/suite.d.ts",
             "import": "./dist/runner/suite.js"
@@ -62,6 +66,7 @@
         "zod": "^4.4.3"
     },
     "devDependencies": {
+        "@google-cloud/aiplatform": "^6.8.1",
         "@types/node": "^22.12.0",
         "tsdown": "^0.22.3",
         "tsx": "^4.22.4",
@@ -71,5 +76,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "packageManager": "pnpm@11.3.0"
+    "packageManager": "pnpm@11.8.0"
 }

package/schemas/eval-interchange-instances.schema.json

@@ -0,0 +1,196 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/alis-build/harness-eval-ts/main/schemas/eval-interchange.schema.json#InstancesJsonlRow",
+  "title": "InstancesJsonlRow",
+  "description": "Type-tagged JSONL row for Vertex EvaluateInstances batching.",
+  "type": "object",
+  "properties": {
+    "messageType": {
+      "$ref": "#/$defs/__schema0"
+    },
+    "caseId": {
+      "$ref": "#/$defs/__schema1"
+    },
+    "repetitionIndex": {
+      "$ref": "#/$defs/__schema2"
+    },
+    "instance": {
+      "$ref": "#/$defs/__schema3"
+    }
+  },
+  "required": [
+    "messageType",
+    "caseId",
+    "repetitionIndex",
+    "instance"
+  ],
+  "additionalProperties": false,
+  "$defs": {
+    "__schema0": {
+      "type": "string",
+      "description": "Vertex protobuf message type name."
+    },
+    "__schema1": {
+      "type": "string",
+      "description": "Test case id."
+    },
+    "__schema2": {
+      "type": "integer",
+      "minimum": -9007199254740991,
+      "maximum": 9007199254740991,
+      "description": "Repetition index."
+    },
+    "__schema3": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/TrajectoryPairInstanceJson"
+        },
+        {
+          "$ref": "#/$defs/TrajectorySingleToolUseInstanceJson"
+        },
+        {
+          "$ref": "#/$defs/EvaluationInstanceJson"
+        }
+      ],
+      "description": "Protojson instance payload."
+    },
+    "TrajectoryPairInstanceJson": {
+      "type": "object",
+      "properties": {
+        "predictedTrajectory": {
+          "$ref": "#/$defs/__schema4"
+        },
+        "referenceTrajectory": {
+          "$ref": "#/$defs/__schema8"
+        }
+      },
+      "required": [
+        "predictedTrajectory",
+        "referenceTrajectory"
+      ],
+      "additionalProperties": false,
+      "title": "TrajectoryPairInstanceJson",
+      "description": "Shared shape for Trajectory*Match/Precision/Recall instances."
+    },
+    "__schema4": {
+      "description": "Predicted tool-call trajectory.",
+      "$ref": "#/$defs/ProtojsonTrajectory"
+    },
+    "__schema5": {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/ProtojsonToolCall"
+      },
+      "description": "Ordered tool calls in the trajectory."
+    },
+    "ProtojsonToolCall": {
+      "type": "object",
+      "properties": {
+        "toolName": {
+          "$ref": "#/$defs/__schema6"
+        },
+        "toolInput": {
+          "$ref": "#/$defs/__schema7"
+        }
+      },
+      "required": [
+        "toolName",
+        "toolInput"
+      ],
+      "additionalProperties": false,
+      "title": "ProtojsonToolCall",
+      "description": "Tool call in Vertex EvaluationService wire format."
+    },
+    "__schema6": {
+      "type": "string",
+      "description": "Tool name as emitted by the agent."
+    },
+    "__schema7": {
+      "type": "string",
+      "description": "JSON-serialized tool arguments (Vertex wire format)."
+    },
+    "ProtojsonTrajectory": {
+      "type": "object",
+      "properties": {
+        "toolCalls": {
+          "$ref": "#/$defs/__schema5"
+        }
+      },
+      "required": [
+        "toolCalls"
+      ],
+      "additionalProperties": false,
+      "title": "ProtojsonTrajectory",
+      "description": "Vertex Trajectory message wire format."
+    },
+    "__schema8": {
+      "description": "Reference tool-call trajectory.",
+      "$ref": "#/$defs/ProtojsonTrajectory"
+    },
+    "TrajectorySingleToolUseInstanceJson": {
+      "type": "object",
+      "properties": {
+        "predictedTrajectory": {
+          "$ref": "#/$defs/__schema9"
+        }
+      },
+      "required": [
+        "predictedTrajectory"
+      ],
+      "additionalProperties": false,
+      "title": "TrajectorySingleToolUseInstanceJson",
+      "description": "Vertex TrajectorySingleToolUseInstance wire format."
+    },
+    "__schema9": {
+      "description": "Predicted tool-call trajectory.",
+      "$ref": "#/$defs/ProtojsonTrajectory"
+    },
+    "EvaluationInstanceJson": {
+      "type": "object",
+      "properties": {
+        "prompt": {
+          "$ref": "#/$defs/__schema10"
+        },
+        "response": {
+          "$ref": "#/$defs/__schema13"
+        },
+        "reference": {
+          "$ref": "#/$defs/__schema14"
+        }
+      },
+      "additionalProperties": false,
+      "title": "EvaluationInstanceJson",
+      "description": "Vertex EvaluationInstance wire format (agentEvalData omitted in v1)."
+    },
+    "__schema10": {
+      "description": "Eval prompt.",
+      "$ref": "#/$defs/InstanceData"
+    },
+    "InstanceData": {
+      "type": "object",
+      "properties": {
+        "text": {
+          "$ref": "#/$defs/__schema11"
+        }
+      },
+      "additionalProperties": false,
+      "title": "InstanceData",
+      "description": "EvaluationInstance prompt/response/reference text wrapper."
+    },
+    "__schema11": {
+      "description": "Plain text instance data.",
+      "$ref": "#/$defs/__schema12"
+    },
+    "__schema12": {
+      "type": "string"
+    },
+    "__schema13": {
+      "description": "Final agent response.",
+      "$ref": "#/$defs/InstanceData"
+    },
+    "__schema14": {
+      "description": "Reference answer text.",
+      "$ref": "#/$defs/InstanceData"
+    }
+  }
+}