@code-yeongyu/senpi-codemode 2026.8.14 → 2026.8.16

package/CHANGELOG.md

@@ -12,6 +12,20 @@
 
 ### Removed
 
+## [2026.8.16] - 2026-08-16
+
+### Breaking Changes
+
+### Added
+
+- Published one versioned `senpi.eval.execution` event per settled eval cell: the in-process bus receives bounded rich call details, while the external RPC projection exposes only byte-capped timing/count metadata for safe OMO analytics; total wall time, kernel runtime, pending calls, exact aggregate totals, and overflow accounting are reported separately ([#897](https://github.com/code-yeongyu/senpi/pull/897)).
+
+### Changed
+
+### Fixed
+
+### Removed
+
 ## [2026.8.14] - 2026-08-14
 
 ### Breaking Changes

package/README.md

@@ -20,6 +20,11 @@ task-tool names are known.
 - Loopback, bearer-authenticated kernel bridge with bounded JSONL frames.
 - Structured status events for file operations, environment access, phases,
   bridge activity, and delegated task progress.
+- One versioned `senpi.eval.execution` event at terminal cell settlement. The
+  in-process event bus receives bounded per-call arguments and result previews
+  for extension-owned consumers; external RPC clients receive a 32 KiB-capped
+  metadata-only projection with wall time, kernel time, exact call counts,
+  pending-call counts, and bounded per-tool aggregates.
 - Bounded streaming output with head/tail previews, column clamping, and
   session-adjacent spill files for large streams.
 - TUI and HTML-export rendering for syntax-highlighted cells, status rows,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@code-yeongyu/senpi-codemode",
-	"version": "2026.8.14",
+	"version": "2026.8.16",
 	"description": "Source-only senpi extension package for codemode evaluation tools",
 	"type": "module",
 	"main": "./src/index.ts",
@@ -30,14 +30,14 @@
 	},
 	"dependencies": {
 		"@babel/parser": "8.0.4",
-		"@earendil-works/pi-ai": "npm:@code-yeongyu/senpi-ai@2026.8.14",
+		"@earendil-works/pi-ai": "npm:@code-yeongyu/senpi-ai@2026.8.16",
 		"typebox": "1.3.8"
 	},
 	"peerDependencies": {
-		"@code-yeongyu/senpi": "2026.8.14"
+		"@code-yeongyu/senpi": "2026.8.16"
 	},
 	"devDependencies": {
-		"@code-yeongyu/senpi": "2026.8.14"
+		"@code-yeongyu/senpi": "2026.8.16"
 	},
 	"keywords": [
 		"senpi",

package/src/index.ts

@@ -17,6 +17,11 @@ import type { CodemodeSessionManager, CreateCodemodeSessionManagerOptions } from
 import { SessionManagerProxy } from "./extension/session-manager-proxy.ts";
 import { WAKE_SOURCE_STATE_EVENT, type WakeSourceState } from "./extension/wake-source-state.ts";
 import { EvalDetachedCellManager, type EvalDetachedCellStatusEntry } from "./tool/detached-cell-manager.ts";
+import {
+	EVAL_EXECUTION_EVENT,
+	type EvalExecutionEventPayload,
+	toEvalExecutionRpcPayload,
+} from "./tool/eval-execution-event.ts";
 import { createEvalTool } from "./tool/eval-tool.ts";
 import { renderEvalCall, renderEvalResult } from "./tool/render.ts";
 
@@ -39,8 +44,10 @@ export interface CodemodeExtensionAPI {
 	getActiveTools(): string[];
 	getAllTools(): readonly EvalSchemaToolInfo[];
 	sendUserMessage(content: string, options?: { deliverAs?: "steer" | "followUp" }): void;
-	/** Optional host event bus; a host without one turns wake-source emission into a harmless no-op. */
+	/** Optional host event bus; a host without one turns extension event emission into a harmless no-op. */
 	events?: { emit(name: string, data: unknown): void };
+	/** Optional host RPC surface for forwarding extension-owned events to connected clients. */
+	rpc?: { emit(name: string, data: unknown): void };
 }
 
 export interface SenpiCodemodeOptions {
@@ -90,6 +97,11 @@ export default function senpiCodemode(pi: CodemodeExtensionAPI, options: SenpiCo
 		modelId: string | undefined,
 		cellManager: EvalDetachedCellManager,
 	): void => {
+		const onCellSettled = (payload: EvalExecutionEventPayload): void => {
+			if (activeCells !== cellManager) return;
+			pi.rpc?.emit(EVAL_EXECUTION_EVENT, toEvalExecutionRpcPayload(payload));
+			pi.events?.emit(EVAL_EXECUTION_EVENT, payload);
+		};
 		pi.registerTool(
 			createEvalTool({
 				enabledLanguages: runtime.enabledLanguages,
@@ -102,6 +114,7 @@ export default function senpiCodemode(pi: CodemodeExtensionAPI, options: SenpiCo
 				artifactsDir: runtime.artifactsDir,
 				cellManager,
 				executionTracker: manager,
+				onCellSettled,
 				renderers,
 				spawns: runtime.spawns,
 				spawnDefaultAgent: runtime.settings.taskTools.task,

package/src/tool/call-capture.ts

@@ -1,6 +1,11 @@
 import { type AgentToolResult, sanitizeTerminalLabel } from "@code-yeongyu/senpi";
+import type { EvalToolCallSummary } from "./types.ts";
 
 export const MAX_ENRICHED_TOOL_CALLS = 30;
+export const MAX_AGGREGATED_TOOL_NAMES = 64;
+export const MAX_CAPTURED_IDENTIFIER_CODE_POINTS = 128;
+export const MAX_CAPTURED_TOOL_NAME_CODE_POINTS = 128;
+export const MAX_RPC_EVENT_BYTES = 32 * 1024;
 
 const MAX_ARGUMENT_STRING_CODE_POINTS = 512;
 const MAX_ARGUMENT_ENTRIES = 32;
@@ -13,6 +18,22 @@ type BoundedValue = {
 	readonly truncated: boolean;
 };
 
+export interface EvalToolCallMetric {
+	readonly name: string;
+	readonly startedAt: number;
+	ok: boolean | undefined;
+	durationMs: number | undefined;
+}
+
+export interface ToolCallCapture {
+	readonly callId: string;
+	readonly args: unknown;
+	readonly startedAt: number;
+	readonly metric: EvalToolCallMetric;
+	readonly includeDetails: boolean;
+	readonly argsTruncated?: true;
+}
+
 export function capCodePoints(text: string, max: number): string {
 	let end = 0;
 	let count = 0;
@@ -83,6 +104,49 @@ export function boundToolCallArgs(args: unknown): { args: unknown; truncated: bo
 	}
 }
 
+export function createToolCallMetric(name: string, startedAt: number): EvalToolCallMetric {
+	return {
+		name: capCodePoints(name, MAX_CAPTURED_TOOL_NAME_CODE_POINTS),
+		startedAt,
+		ok: undefined,
+		durationMs: undefined,
+	};
+}
+
+export function settleToolCallMetric(metric: EvalToolCallMetric, ok: boolean, completedAt: number): void {
+	metric.ok = ok;
+	metric.durationMs = Math.max(0, completedAt - metric.startedAt);
+}
+
+export function recordToolCall(
+	toolCalls: EvalToolCallSummary[],
+	ok: boolean,
+	capture: ToolCallCapture,
+	resultPreview: string | undefined,
+	error: string | undefined,
+): void {
+	const completedAt = Date.now();
+	settleToolCallMetric(capture.metric, ok, completedAt);
+	const summary = {
+		name: capture.metric.name,
+		ok,
+		...(error === undefined ? {} : { error: capCodePoints(error, 512) }),
+		...(capture.includeDetails ? { durationMs: completedAt - capture.startedAt } : {}),
+	};
+	const enrichedCount = toolCalls.filter((toolCall) => toolCall.callId !== undefined).length;
+	if (!capture.includeDetails || enrichedCount >= MAX_ENRICHED_TOOL_CALLS) {
+		toolCalls.push(summary);
+		return;
+	}
+	toolCalls.push({
+		...summary,
+		callId: capture.callId,
+		args: capture.args,
+		...(capture.argsTruncated === true ? { argsTruncated: true } : {}),
+		...(resultPreview === undefined ? {} : { resultPreview }),
+	});
+}
+
 export function toolCallResultPreview(result: AgentToolResult<unknown>): string | undefined {
 	for (const part of result.content) {
 		if (part.type !== "text") continue;

package/src/tool/cell-handler.ts

@@ -1,5 +1,6 @@
 import { type AgentToolResult, type ExtensionContext, sanitizeTerminalLabel } from "@code-yeongyu/senpi";
 import type { KernelToHostMessage } from "../bridge/protocol.ts";
+import { RESERVED_SCHEMA_TOOL } from "../bridge/reserved.ts";
 import type { AgentExecuteTool } from "../bridges/agent-bridge.ts";
 import { isReservedToolName, runReservedTool } from "../bridges/reserved-dispatch.ts";
 import type { EvalSchemaToolInfo } from "../bridges/schema-bridge.ts";
@@ -7,7 +8,15 @@ import { appendSchemaHint } from "../bridges/schema-hint.ts";
 import type { CompletionRequest, CompletionResult } from "../completion/handler.ts";
 import { handleCompletionToolCall } from "../completion/tool-bridge.ts";
 import type { ResolvedCodemodeSettings } from "../config/settings.ts";
-import { boundToolCallArgs, capCodePoints, MAX_ENRICHED_TOOL_CALLS, toolCallResultPreview } from "./call-capture.ts";
+import {
+	boundToolCallArgs,
+	capCodePoints,
+	createToolCallMetric,
+	MAX_CAPTURED_IDENTIFIER_CODE_POINTS,
+	recordToolCall,
+	type ToolCallCapture,
+	toolCallResultPreview,
+} from "./call-capture.ts";
 import { CellResultBuilder, type CellState } from "./cell-runtime.ts";
 import { type EvalImageResizer, marshalToolResult, toolResultIsError } from "./image.ts";
 import { upsertStatusEvent } from "./status-events.ts";
@@ -22,13 +31,6 @@ type ResolvedToolReply = {
 	readonly errorText?: string;
 };
 
-type ToolCallEnrichment = {
-	readonly callId: string;
-	readonly args: unknown;
-	readonly startedAt: number;
-	readonly argsTruncated?: true;
-};
-
 export interface CellBridgeRuntime {
 	readonly executeTool: AgentExecuteTool;
 	readonly listTools?: () => readonly EvalSchemaToolInfo[];
@@ -111,9 +113,21 @@ export class CellHandler {
 	}
 
 	async #handleToolCall(message: Extract<KernelToHostMessage, { type: "tool-call" }>): Promise<void> {
+		const startedAt = Date.now();
+		const metric = createToolCallMetric(message.toolName, startedAt);
+		this.#state.toolCallMetrics.push(metric);
+		const capturedArgs = boundToolCallArgs(message.args);
+		const capture: ToolCallCapture = {
+			callId: capCodePoints(message.callId, MAX_CAPTURED_IDENTIFIER_CODE_POINTS),
+			args: capturedArgs.args,
+			startedAt,
+			metric,
+			includeDetails: message.toolName !== RESERVED_SCHEMA_TOOL,
+			...(capturedArgs.truncated ? { argsTruncated: true } : {}),
+		};
 		if (message.toolName === "eval") {
 			const error = "recursive eval is not allowed";
-			this.#state.toolCalls.push({ name: message.toolName, ok: false, error });
+			recordToolCall(this.#state.toolCalls, false, capture, undefined, error);
 			this.#kernel.deliverToolReply({
 				type: "tool-reply",
 				callId: message.callId,
@@ -123,20 +137,24 @@ export class CellHandler {
 			return;
 		}
 		if (isReservedToolName(message.toolName)) {
-			await this.#deliverToolReply(message, async () => ({
-				value: await runReservedTool(message.toolName, {
-					callId: message.callId,
-					args: message.args,
-					executeTool: this.#runtime.executeTool,
-					taskToolName: this.#runtime.settings.taskTools.task,
-					taskOutputToolName: this.#runtime.settings.taskTools.output,
-					listTools: this.#runtime.listTools,
-					signal: this.#state.signal,
-					emitStatus: (event) => this.#recordStatus(event),
-					marshalToolResult,
+			await this.#deliverToolReply(
+				message,
+				async () => ({
+					value: await runReservedTool(message.toolName, {
+						callId: message.callId,
+						args: message.args,
+						executeTool: this.#runtime.executeTool,
+						taskToolName: this.#runtime.settings.taskTools.task,
+						taskOutputToolName: this.#runtime.settings.taskTools.output,
+						listTools: this.#runtime.listTools,
+						signal: this.#state.signal,
+						emitStatus: (event) => this.#recordStatus(event),
+						marshalToolResult,
+					}),
+					toolCallOk: true,
 				}),
-				toolCallOk: true,
-			}));
+				capture,
+			);
 			return;
 		}
 		if (message.toolName === "completion" && this.#runtime.complete) {
@@ -148,16 +166,10 @@ export class CellHandler {
 				isActive: () => this.#state.active,
 			});
 			if (!this.#state.active) return;
-			this.#state.toolCalls.push(
-				result.ok
-					? { name: message.toolName, ok: true }
-					: { name: message.toolName, ok: false, error: result.error },
-			);
+			recordToolCall(this.#state.toolCalls, result.ok, capture, undefined, result.ok ? undefined : result.error);
 			this.#resultBuilder.emitUpdate(false);
 			return;
 		}
-		const capturedArgs = boundToolCallArgs(message.args);
-		const startedAt = Date.now();
 		await this.#deliverToolReply(
 			message,
 			async () => {
@@ -185,24 +197,19 @@ export class CellHandler {
 					...(errorText === undefined ? {} : { errorText }),
 				};
 			},
-			{
-				callId: message.callId,
-				args: capturedArgs.args,
-				startedAt,
-				...(capturedArgs.truncated ? { argsTruncated: true } : {}),
-			},
+			capture,
 		);
 	}
 
 	async #deliverToolReply(
 		message: Extract<KernelToHostMessage, { type: "tool-call" }>,
 		resolve: () => Promise<ResolvedToolReply>,
-		enrich?: ToolCallEnrichment,
+		capture: ToolCallCapture,
 	): Promise<void> {
 		try {
 			const reply = await resolve();
 			if (!this.#state.active) return;
-			this.#pushToolCall(message.toolName, reply.toolCallOk, enrich, reply.resultPreview, reply.errorText);
+			recordToolCall(this.#state.toolCalls, reply.toolCallOk, capture, reply.resultPreview, reply.errorText);
 			this.#kernel.deliverToolReply({ type: "tool-reply", callId: message.callId, ok: true, value: reply.value });
 		} catch (error) {
 			if (!this.#state.active) return;
@@ -211,7 +218,7 @@ export class CellHandler {
 				message.toolName,
 				this.#toolParameters(message.toolName),
 			);
-			this.#pushToolCall(message.toolName, false, enrich, undefined, text);
+			recordToolCall(this.#state.toolCalls, false, capture, undefined, text);
 			this.#kernel.deliverToolReply({
 				type: "tool-reply",
 				callId: message.callId,
@@ -222,29 +229,6 @@ export class CellHandler {
 		this.#resultBuilder.emitUpdate(false);
 	}
 
-	#pushToolCall(
-		name: string,
-		ok: boolean,
-		enrich: ToolCallEnrichment | undefined,
-		resultPreview: string | undefined,
-		error: string | undefined,
-	): void {
-		const summary = { name, ok, ...(error === undefined ? {} : { error }) };
-		const enrichedCount = this.#state.toolCalls.filter((toolCall) => toolCall.callId !== undefined).length;
-		if (enrich === undefined || enrichedCount >= MAX_ENRICHED_TOOL_CALLS) {
-			this.#state.toolCalls.push(summary);
-			return;
-		}
-		this.#state.toolCalls.push({
-			...summary,
-			callId: enrich.callId,
-			args: enrich.args,
-			durationMs: Date.now() - enrich.startedAt,
-			...(enrich.argsTruncated === true ? { argsTruncated: true } : {}),
-			...(resultPreview === undefined ? {} : { resultPreview }),
-		});
-	}
-
 	#toolParameters(toolName: string): unknown {
 		return this.#runtime.listTools?.().find((tool) => tool.name === toolName)?.parameters;
 	}

package/src/tool/cell-runtime.ts

@@ -1,5 +1,6 @@
 import type { AgentToolResult, AgentToolUpdateCallback, ExtensionContext } from "@code-yeongyu/senpi";
 import type { KernelToHostMessage } from "../bridge/protocol.ts";
+import type { EvalToolCallMetric } from "./call-capture.ts";
 import { type EvalImageResizer, EvalOutputCollector, type EvalOutputResult } from "./image.ts";
 import type { EvalStatusEvent, EvalToolDetails, EvalToolInput } from "./types.ts";
 
@@ -9,14 +10,17 @@ type ToolCall = EvalToolDetails["toolCalls"] extends readonly (infer Item)[] ? I
 
 export interface CellState {
 	readonly input: EvalToolInput;
+	readonly startedAt: number;
 	readonly signal: AbortSignal;
 	readonly onUpdate: AgentToolUpdateCallback<EvalToolDetails> | undefined;
 	readonly toolCalls: ToolCall[];
+	readonly toolCallMetrics: EvalToolCallMetric[];
 	readonly pendingBridgeCalls: Promise<void>[];
 	readonly statusEvents: EvalStatusEvent[];
 	active: boolean;
 	output: string;
 	phase: string | undefined;
+	error: string | undefined;
 	durationMs: number;
 	status: "pending" | "running" | "complete" | "error";
 }
@@ -70,6 +74,7 @@ export class CellResultBuilder {
 			if (result.valueRepr) this.#output.push(`${result.valueRepr}\n`);
 			this.#state.status = "complete";
 		} else {
+			this.#state.error = result.error.message;
 			this.#output.push(`${result.error.message}\n`);
 			this.#state.status = "error";
 		}
@@ -77,6 +82,7 @@ export class CellResultBuilder {
 	}
 
 	async finalizeCancellation(error: Error): Promise<AgentToolResult<EvalToolDetails>> {
+		this.#state.error = error.message;
 		this.#output.push(`${error.message}\n`);
 		this.#state.status = "error";
 		return await this.#finish(true);

package/src/tool/eval-execution-event.ts

@@ -0,0 +1,201 @@
+import type { AgentToolResult } from "@code-yeongyu/senpi";
+import {
+	capCodePoints,
+	MAX_AGGREGATED_TOOL_NAMES,
+	MAX_CAPTURED_IDENTIFIER_CODE_POINTS,
+	MAX_ENRICHED_TOOL_CALLS,
+	MAX_RPC_EVENT_BYTES,
+} from "./call-capture.ts";
+import type { CellState } from "./cell-runtime.ts";
+import type { EvalLanguage, EvalToolCallSummary, EvalToolDetails } from "./types.ts";
+
+export const EVAL_EXECUTION_EVENT = "senpi.eval.execution";
+
+export interface EvalToolAggregate {
+	readonly count: number;
+	readonly totalDurationMs: number;
+	readonly okCount: number;
+	readonly errorCount: number;
+	readonly pendingCount: number;
+}
+
+export type EvalToolAggregates = Record<string, EvalToolAggregate>;
+
+interface EvalExecutionBasePayload {
+	readonly version: 1;
+	readonly cellId: string;
+	readonly language: EvalLanguage;
+	readonly ok: boolean;
+	readonly startedAt: number;
+	readonly completedAt: number;
+	readonly durationMs: number;
+	readonly kernelDurationMs?: number;
+	readonly detached: boolean;
+	readonly toolCallCount: number;
+	readonly pendingToolCallCount: number;
+	readonly distinctToolsCalled: readonly string[];
+	readonly toolAggregates: EvalToolAggregates;
+	readonly toolAggregatesTruncated: boolean;
+	readonly toolAggregateOverflow?: EvalToolAggregate;
+}
+
+export interface EvalExecutionEventPayload extends EvalExecutionBasePayload {
+	readonly detailLevel: "full";
+	readonly error?: string;
+	readonly toolCalls: readonly EvalToolCallSummary[];
+}
+
+export interface EvalExecutionRpcToolCallSummary {
+	readonly name: string;
+	readonly ok: boolean;
+	readonly durationMs?: number;
+}
+
+export interface EvalExecutionRpcPayload extends EvalExecutionBasePayload {
+	readonly detailLevel: "metadata";
+	readonly rpcTruncated: boolean;
+	readonly toolCalls: readonly EvalExecutionRpcToolCallSummary[];
+}
+
+export type EvalExecutionSettleOutcome =
+	| { readonly result: AgentToolResult<EvalToolDetails> }
+	| { readonly error: unknown };
+
+export interface BuildEvalExecutionEventPayloadOptions {
+	readonly cellId: string;
+	readonly state: CellState;
+	readonly outcome: EvalExecutionSettleOutcome;
+	readonly completedAt: number;
+	readonly detached: boolean;
+}
+
+export function buildEvalExecutionEventPayload(
+	options: BuildEvalExecutionEventPayloadOptions,
+): EvalExecutionEventPayload {
+	const { cellId, state, outcome, completedAt, detached } = options;
+	const result = "result" in outcome ? outcome.result : undefined;
+	const ok = result?.details.isError !== true && !("error" in outcome);
+	const aggregateState = aggregateToolCalls(state, completedAt);
+	const error = ok ? undefined : settleError(state, outcome);
+	return {
+		version: 1,
+		detailLevel: "full",
+		cellId: capCodePoints(cellId, MAX_CAPTURED_IDENTIFIER_CODE_POINTS),
+		language: state.input.language,
+		ok,
+		...(error === undefined ? {} : { error: capCodePoints(error, 512) }),
+		startedAt: state.startedAt,
+		completedAt,
+		durationMs: Math.max(0, completedAt - state.startedAt),
+		...(result === undefined ? {} : { kernelDurationMs: result.details.durationMs }),
+		detached,
+		toolCallCount: state.toolCallMetrics.length,
+		pendingToolCallCount: aggregateState.pendingCount,
+		toolCalls: state.toolCalls.slice(0, MAX_ENRICHED_TOOL_CALLS),
+		distinctToolsCalled: [...aggregateState.aggregates.keys()],
+		toolAggregates: Object.fromEntries(aggregateState.aggregates),
+		toolAggregatesTruncated: aggregateState.overflow !== undefined,
+		...(aggregateState.overflow === undefined ? {} : { toolAggregateOverflow: aggregateState.overflow }),
+	};
+}
+
+export function toEvalExecutionRpcPayload(payload: EvalExecutionEventPayload): EvalExecutionRpcPayload {
+	const candidate: EvalExecutionRpcPayload = {
+		version: payload.version,
+		detailLevel: "metadata",
+		rpcTruncated: false,
+		cellId: payload.cellId,
+		language: payload.language,
+		ok: payload.ok,
+		startedAt: payload.startedAt,
+		completedAt: payload.completedAt,
+		durationMs: payload.durationMs,
+		...(payload.kernelDurationMs === undefined ? {} : { kernelDurationMs: payload.kernelDurationMs }),
+		detached: payload.detached,
+		toolCallCount: payload.toolCallCount,
+		pendingToolCallCount: payload.pendingToolCallCount,
+		toolCalls: payload.toolCalls.map((call) => ({
+			name: call.name,
+			ok: call.ok,
+			...(call.durationMs === undefined ? {} : { durationMs: call.durationMs }),
+		})),
+		distinctToolsCalled: payload.distinctToolsCalled,
+		toolAggregates: payload.toolAggregates,
+		toolAggregatesTruncated: payload.toolAggregatesTruncated,
+		...(payload.toolAggregateOverflow === undefined ? {} : { toolAggregateOverflow: payload.toolAggregateOverflow }),
+	};
+	if (serializedBytes(candidate) <= MAX_RPC_EVENT_BYTES) return candidate;
+	return {
+		...candidate,
+		rpcTruncated: true,
+		toolCalls: [],
+		distinctToolsCalled: [],
+		toolAggregates: {},
+		toolAggregatesTruncated: true,
+		toolAggregateOverflow: totalAggregate(payload),
+	};
+}
+
+function aggregateToolCalls(
+	state: CellState,
+	completedAt: number,
+): {
+	readonly aggregates: Map<string, EvalToolAggregate>;
+	readonly overflow: EvalToolAggregate | undefined;
+	readonly pendingCount: number;
+} {
+	const aggregates = new Map<string, EvalToolAggregate>();
+	let overflow: EvalToolAggregate | undefined;
+	let pendingCount = 0;
+	for (const metric of state.toolCallMetrics) {
+		const item: EvalToolAggregate = {
+			count: 1,
+			totalDurationMs: metric.durationMs ?? Math.max(0, completedAt - metric.startedAt),
+			okCount: metric.ok === true ? 1 : 0,
+			errorCount: metric.ok === false ? 1 : 0,
+			pendingCount: metric.ok === undefined ? 1 : 0,
+		};
+		pendingCount += item.pendingCount;
+		const existing = aggregates.get(metric.name);
+		if (existing !== undefined) {
+			aggregates.set(metric.name, addAggregate(existing, item));
+			continue;
+		}
+		if (aggregates.size < MAX_AGGREGATED_TOOL_NAMES) {
+			aggregates.set(metric.name, item);
+			continue;
+		}
+		overflow = overflow === undefined ? item : addAggregate(overflow, item);
+	}
+	return { aggregates, overflow, pendingCount };
+}
+
+function addAggregate(left: EvalToolAggregate, right: EvalToolAggregate): EvalToolAggregate {
+	return {
+		count: left.count + right.count,
+		totalDurationMs: left.totalDurationMs + right.totalDurationMs,
+		okCount: left.okCount + right.okCount,
+		errorCount: left.errorCount + right.errorCount,
+		pendingCount: left.pendingCount + right.pendingCount,
+	};
+}
+
+function totalAggregate(payload: EvalExecutionEventPayload): EvalToolAggregate {
+	let total: EvalToolAggregate = { count: 0, totalDurationMs: 0, okCount: 0, errorCount: 0, pendingCount: 0 };
+	for (const aggregate of Object.values(payload.toolAggregates)) total = addAggregate(total, aggregate);
+	if (payload.toolAggregateOverflow !== undefined) total = addAggregate(total, payload.toolAggregateOverflow);
+	return total;
+}
+
+function serializedBytes(value: unknown): number {
+	return new TextEncoder().encode(JSON.stringify(value)).byteLength;
+}
+
+function settleError(state: CellState, outcome: EvalExecutionSettleOutcome): string | undefined {
+	if (state.error !== undefined) return state.error;
+	if ("error" in outcome) return outcome.error instanceof Error ? outcome.error.message : String(outcome.error);
+	for (const part of outcome.result.content) {
+		if (part.type === "text" && part.text.length > 0) return part.text.trimEnd();
+	}
+	return undefined;
+}

package/src/tool/eval-tool-options.ts

@@ -5,6 +5,7 @@ import type { ResolvedCodemodeSettings } from "../config/settings.ts";
 import type { EvalExecutionTracker } from "../extension/session-manager.ts";
 import type { EvalTimeoutFactory } from "./cell-execution.ts";
 import type { EvalDetachedCellManager } from "./detached-cell-manager.ts";
+import type { EvalExecutionEventPayload } from "./eval-execution-event.ts";
 import type { EvalImageResizer } from "./image.ts";
 import type {
 	EnabledEvalLanguages,
@@ -29,6 +30,7 @@ export interface CreateEvalToolOptions {
 	readonly imageResizer?: EvalImageResizer;
 	readonly executionTracker?: EvalExecutionTracker;
 	readonly cellManager?: EvalDetachedCellManager;
+	readonly onCellSettled?: (payload: EvalExecutionEventPayload) => void;
 	readonly timeoutFactory?: EvalTimeoutFactory;
 	readonly proxyExecutor?: (params: EvalToolInput, signal?: AbortSignal) => Promise<AgentToolResult<EvalToolDetails>>;
 	readonly renderers?: Pick<ToolDefinition<EvalInputSchema, EvalToolDetails>, "renderCall" | "renderResult">;

package/src/tool/eval-tool.ts

@@ -8,6 +8,7 @@ import { abortError, CellExecution, defaultTimeoutFactory } from "./cell-executi
 import { CellHandler, type CellState } from "./cell-handler.ts";
 import { EvalDetachedCellManager } from "./detached-cell-manager.ts";
 import { detachedKernelBusyError, executeEvalControl, resultAfterDetach } from "./detached-eval-result.ts";
+import { buildEvalExecutionEventPayload, type EvalExecutionSettleOutcome } from "./eval-execution-event.ts";
 import { clampEvalSummary, evalTimeoutBehavior, isEvalControlRequest, parseEvalRequest } from "./eval-request.ts";
 import type { CreateEvalToolOptions, EvalCellInvocation } from "./eval-tool-options.ts";
 import { describeTimeoutState } from "./interrupt-note.ts";
@@ -99,18 +100,22 @@ async function runEvalCell(
 	const bridgeContext: ExtensionContext = { ...invocation.ctx, signal: cellSignal };
 	const state: CellState = {
 		input: invocation.input,
+		startedAt: Date.now(),
 		signal: cellSignal,
 		onUpdate: invocation.onUpdate,
 		toolCalls: [],
+		toolCallMetrics: [],
 		pendingBridgeCalls: [],
 		statusEvents: [],
 		active: true,
 		output: "",
 		phase: undefined,
+		error: undefined,
 		durationMs: 0,
 		status: "pending",
 	};
 	const cell = cellManager.create(invocation.cellId, invocation.input);
+	let detached = false;
 	let execution: CellExecution;
 	execution = new CellExecution({
 		callerSignal: invocation.signal,
@@ -119,6 +124,7 @@ async function runEvalCell(
 		timeoutFactory: options.timeoutFactory ?? defaultTimeoutFactory,
 		onTimeout: (error) => {
 			if (timeoutBehavior === "detach" && cellManager.detach(cell)) {
+				detached = true;
 				execution.detach();
 				return;
 			}
@@ -139,13 +145,29 @@ async function runEvalCell(
 		bridgeContext,
 		bridgeAbortController,
 	);
+	let settleEventEmitted = false;
+	const emitSettled = (outcome: EvalExecutionSettleOutcome): void => {
+		if (settleEventEmitted) return;
+		settleEventEmitted = true;
+		options.onCellSettled?.(
+			buildEvalExecutionEventPayload({
+				cellId: invocation.cellId,
+				state,
+				outcome,
+				completedAt: Date.now(),
+				detached,
+			}),
+		);
+	};
 	const finalized = running.then(
 		(result) => {
 			cellManager.complete(cell, result);
+			emitSettled({ result });
 			return result;
 		},
 		(error: unknown) => {
 			cellManager.fail(cell, error instanceof Error ? error : new Error(String(error)));
+			emitSettled({ error });
 			throw error;
 		},
 	);