@gajae-code/coding-agent 0.12.10 → 0.12.11

package/src/tools/computer.ts

@@ -99,7 +99,7 @@ export interface ComputerScreenshotDetails {
 	originX?: number;
 	originY?: number;
 	displayEpoch?: number;
-	captureId?: string;
+	captureId?: number;
 	pngBytes?: number;
 	path?: string;
 }
@@ -119,12 +119,50 @@ export interface ComputerToolDetails {
 	keys?: string[];
 	ms?: number;
 	screenshot?: ComputerScreenshotDetails;
+	primaryError?: { code: string; message: string };
 	supervisor?: string;
 	steps?: ComputerToolDetails[];
 	meta?: OutputMeta;
 }
 
+type NativeBatchAction = {
+	action: "screenshot" | "click" | "double_click" | "move" | "drag" | "scroll" | "type" | "keypress" | "wait";
+	x?: number;
+	y?: number;
+	toX?: number;
+	toY?: number;
+	scrollX?: number;
+	scrollY?: number;
+	button?: string;
+	text?: string;
+	keys?: string[];
+	ms?: number;
+	timeoutMs?: number;
+	timeoutGroup?: number;
+};
+
+type NativeBatchStepResult = {
+	index: number;
+	action: string;
+	screenshot?: NativeScreenshot;
+};
+
+type NativeBatchResult = {
+	results: NativeBatchStepResult[];
+	failureCode?: string;
+	failureIndex?: number;
+	failureMessage?: string;
+	primaryFailureCode?: string;
+	primaryFailureMessage?: string;
+};
+
 type NativeController = {
+	executeBatch?: (
+		expectedEpoch: number | undefined,
+		actions: NativeBatchAction[],
+		timeoutMs?: number,
+		signal?: AbortSignal,
+	) => Promise<NativeBatchResult> | NativeBatchResult;
 	screenshot?: () => Promise<NativeScreenshot> | NativeScreenshot;
 	click?: (expectedEpoch: number | undefined, x: number, y: number, button?: string) => void;
 	doubleClick?: (expectedEpoch: number | undefined, x: number, y: number, button?: string) => void;
@@ -145,7 +183,7 @@ type NativeScreenshot = {
 	originX?: number;
 	originY?: number;
 	displayEpoch?: number;
-	captureId?: string;
+	captureId?: number;
 };
 
 export type ComputerControllerFactory = () => NativeController;
@@ -183,7 +221,7 @@ const COMPUTER_INLINE_SCREENSHOT_PROVIDER_MAX_BYTES = 5 * 1024 * 1024;
 const COMPUTER_INLINE_SCREENSHOT_JPEG_QUALITY = 70;
 
 export function setComputerControllerFactoryForTests(factory: ComputerControllerFactory | undefined): void {
-	controllerFactory = factory ?? createNativeComputerController;
+	controllerFactory = factory ? () => withLegacyBatchAdapterForTests(factory()) : createNativeComputerController;
 }
 
 export function setComputerPlatformForTests(platform: NodeJS.Platform | undefined): void {
@@ -298,6 +336,10 @@ export class ComputerTool implements AgentTool<typeof computerSchema, ComputerTo
 				if (batchResult.failedStep) {
 					details.code = batchResult.failedStep.code;
 					details.message = batchResult.failedStep.message;
+					if (batchResult.primaryError) {
+						details.primaryError = batchResult.primaryError;
+						details.message += ` Primary failure: ${batchResult.primaryError.code}: ${batchResult.primaryError.message}`;
+					}
 					if (batchResult.screenshotSource !== undefined) {
 						await persistScreenshotFallback(batchResult.screenshotSource, details.screenshot, this.session);
 					}
@@ -320,6 +362,57 @@ export class ComputerTool implements AgentTool<typeof computerSchema, ComputerTo
 							.done()
 					: toolResult(details).text(details.message).done();
 			}
+			if (params.action !== "screenshot") {
+				const singleResult = await dispatchBatchComputerActions(
+					controller,
+					[params],
+					timeoutMs,
+					hotkey,
+					latestScreenshotContexts.get(this.session),
+					false,
+					Boolean(this.session.settings.get("computer.autoScreenshot")),
+					signal,
+					deadline,
+				);
+				if (singleResult.screenshot) {
+					details.screenshot = singleResult.screenshot;
+					rememberLatestScreenshot(this.session, singleResult.screenshot);
+				}
+				if (singleResult.failedStep) {
+					details.status = singleResult.failedStep.code === COMPUTER_DISABLED_CODE ? "disabled" : "error";
+					details.code = singleResult.failedStep.code;
+					details.message = singleResult.failedStep.message;
+					if (singleResult.primaryError) {
+						details.primaryError = singleResult.primaryError;
+						details.message += ` Primary failure: ${singleResult.primaryError.code}: ${singleResult.primaryError.message}`;
+					}
+					if (singleResult.screenshotSource !== undefined) {
+						await persistScreenshotFallback(singleResult.screenshotSource, details.screenshot, this.session);
+					}
+					await writeComputerAuditLog(this.session, details);
+					return {
+						...toolResult(details).text(`${details.code}: ${details.message}`).done(),
+						isError: true,
+					};
+				}
+				details.status = "success";
+				details.message = describeComputerSuccess(details);
+				if (singleResult.screenshotSource !== undefined) {
+					await persistScreenshotFallback(singleResult.screenshotSource, details.screenshot, this.session);
+					details.message = describeComputerSuccess(details);
+				}
+				const image = await inlineImageContentFromNativeResult(
+					singleResult.screenshotSource,
+					details,
+					this.session,
+				);
+				await writeComputerAuditLog(this.session, details);
+				return image
+					? toolResult(details)
+							.content([{ type: "text", text: details.message }, image])
+							.done()
+					: toolResult(details).text(details.message).done();
+			}
 			let result = await dispatchComputerAction(
 				controller,
 				params,
@@ -509,6 +602,85 @@ async function runComputerOperation<T>(
 	}
 }
 
+function withLegacyBatchAdapterForTests(controller: NativeController): NativeController {
+	if (
+		controller.executeBatch ||
+		(!controller.click &&
+			!controller.doubleClick &&
+			!controller.move &&
+			!controller.drag &&
+			!controller.scroll &&
+			!controller.type &&
+			!controller.keypress &&
+			!controller.wait)
+	) {
+		return controller;
+	}
+	return {
+		...controller,
+		executeBatch: async (expectedEpoch, actions, _timeoutMs, signal) => {
+			const results: NativeBatchStepResult[] = [];
+			for (const [index, action] of actions.entries()) {
+				throwIfAborted(signal);
+				switch (action.action) {
+					case "screenshot":
+						if (!controller.screenshot) missingNativeMethod("screenshot", "screenshot");
+						results.push({ index, action: "screenshot", screenshot: await controller.screenshot() });
+						break;
+					case "click":
+						if (!controller.click) missingNativeMethod("click", "click");
+						controller.click(expectedEpoch, action.x!, action.y!, action.button ?? "left");
+						results.push({ index, action: "click" });
+						break;
+					case "double_click":
+						if (!controller.doubleClick) missingNativeMethod("double_click", "doubleClick");
+						controller.doubleClick(expectedEpoch, action.x!, action.y!, action.button ?? "left");
+						results.push({ index, action: "double_click" });
+						break;
+					case "move":
+						if (!controller.move) missingNativeMethod("move", "move");
+						controller.move(expectedEpoch, action.x!, action.y!);
+						results.push({ index, action: "move" });
+						break;
+					case "drag":
+						if (!controller.drag) missingNativeMethod("drag", "drag");
+						controller.drag(
+							expectedEpoch,
+							action.x!,
+							action.y!,
+							action.toX!,
+							action.toY!,
+							action.button ?? "left",
+						);
+						results.push({ index, action: "drag" });
+						break;
+					case "scroll":
+						if (!controller.scroll) missingNativeMethod("scroll", "scroll");
+						controller.scroll(expectedEpoch, action.x!, action.y!, action.scrollX!, action.scrollY!);
+						results.push({ index, action: "scroll" });
+						break;
+					case "type":
+						if (!controller.type) missingNativeMethod("type", "type");
+						controller.type(undefined, action.text!);
+						results.push({ index, action: "type" });
+						break;
+					case "keypress":
+						if (!controller.keypress) missingNativeMethod("keypress", "keypress");
+						controller.keypress(undefined, action.keys!);
+						results.push({ index, action: "keypress" });
+						break;
+					case "wait":
+						if (!controller.wait) missingNativeMethod("wait", "wait");
+						controller.wait(undefined, action.ms!);
+						results.push({ index, action: "wait" });
+						break;
+				}
+			}
+			return { results };
+		},
+	};
+}
+
 function dispatchComputerAction(
 	controller: NativeController,
 	params: SingleComputerParams,
@@ -572,6 +744,64 @@ interface BatchDispatchResult {
 	screenshot?: ComputerScreenshotDetails;
 	screenshotSource?: unknown;
 	failedStep?: { code: string; message: string };
+	primaryError?: { code: string; message: string };
+}
+
+interface BatchWireStep {
+	action: NativeBatchAction;
+	userIndex?: number;
+	finalScreenshot?: boolean;
+}
+
+function nativeBatchAction(params: SingleComputerParams, timeoutMs: number | undefined): NativeBatchAction {
+	const actionTimeoutMs = stepTimeoutFromParams(params, timeoutMs);
+	switch (params.action) {
+		case "screenshot":
+			return { action: "screenshot", timeoutMs: actionTimeoutMs };
+		case "click":
+		case "double_click":
+		case "move":
+			return {
+				action: params.action,
+				x: params.x,
+				y: params.y,
+				button: params.button,
+				timeoutMs: actionTimeoutMs,
+			};
+		case "drag":
+			return {
+				action: "drag",
+				x: params.x,
+				y: params.y,
+				toX: params.to_x,
+				toY: params.to_y,
+				button: params.button,
+				timeoutMs: actionTimeoutMs,
+			};
+		case "scroll":
+			return {
+				action: "scroll",
+				x: params.x,
+				y: params.y,
+				scrollX: params.scroll_x,
+				scrollY: params.scroll_y,
+				timeoutMs: actionTimeoutMs,
+			};
+		case "type":
+			return { action: "type", text: params.text, timeoutMs: actionTimeoutMs };
+		case "keypress":
+			return { action: "keypress", keys: params.keys, timeoutMs: actionTimeoutMs };
+		case "wait":
+			return { action: "wait", ms: capWaitMs(params.ms, actionTimeoutMs), timeoutMs: actionTimeoutMs };
+	}
+}
+
+function mapNativeBatchFailure(
+	code: string,
+	message: string | undefined,
+	hotkey?: string,
+): { code: string; message: string } {
+	return mapComputerError({ code, message: message ?? `${code}: Computer batch failed.` }, hotkey);
 }
 
 async function dispatchBatchComputerActions(
@@ -585,58 +815,100 @@ async function dispatchBatchComputerActions(
 	signal?: AbortSignal,
 	deadline?: ComputerDeadline,
 ): Promise<BatchDispatchResult> {
-	const steps: ComputerToolDetails[] = [];
+	if (!controller.executeBatch) missingNativeMethod("batch", "executeBatch");
+
+	const steps = actions.map(detailsFromParams);
+	const wireSteps: BatchWireStep[] = [];
+	for (const [userIndex, action] of actions.entries()) {
+		const nativeAction = { ...nativeBatchAction(action, timeoutMs), timeoutGroup: userIndex };
+		wireSteps.push({ action: nativeAction, userIndex });
+		if (action.action !== "screenshot" && (action.include_screenshot === true || autoScreenshot)) {
+			wireSteps.push({
+				action: {
+					action: "screenshot",
+					timeoutMs: nativeAction.timeoutMs,
+					timeoutGroup: nativeAction.timeoutGroup,
+				},
+				userIndex,
+			});
+		}
+	}
+	if (includeBatchScreenshot) {
+		wireSteps.push({ action: { action: "screenshot", timeoutMs }, finalScreenshot: true });
+	}
+
+	throwIfAborted(signal);
+	assertComputerDeadline(deadline);
+	const nativeTimeoutMs = remainingComputerTimeoutMs(deadline);
+	const nativeResult = await controller.executeBatch(
+		expectedEpochFromContext(initialContext),
+		wireSteps.map(step => step.action),
+		nativeTimeoutMs,
+		signal,
+	);
 	let lastScreenshot: ComputerScreenshotDetails | undefined;
 	let lastScreenshotSource: unknown;
-	let context = initialContext;
-	for (const single of actions) {
-		const stepDetails = detailsFromParams(single);
-		try {
-			throwIfAborted(signal);
-			assertComputerDeadline(deadline);
-			const stepTimeoutMs = stepTimeoutFromParams(single, timeoutMs);
-			const stepDeadline = createComputerDeadline(stepTimeoutMs, deadline);
-			let result = await dispatchComputerAction(controller, single, stepDeadline, context, signal);
-			if (single.action !== "screenshot" && (single.include_screenshot === true || autoScreenshot)) {
-				result = await captureScreenshot(controller, stepDeadline, signal);
-			}
-
-			const screenshot = normalizeScreenshot(result);
-			if (screenshot) {
-				stepDetails.screenshot = screenshot;
-				lastScreenshot = screenshot;
-				lastScreenshotSource = result;
-				context = screenshot;
-			}
-			stepDetails.status = "success";
-			stepDetails.message = describeComputerSuccess(stepDetails);
-		} catch (error) {
-			if (error instanceof ToolAbortError) throw error;
-			const mapped = mapComputerError(error, hotkey);
-			stepDetails.status = mapped.code === COMPUTER_DISABLED_CODE ? "disabled" : "error";
-			stepDetails.code = mapped.code;
-			stepDetails.message = mapped.message;
-			steps.push(stepDetails);
-			return {
-				steps,
-				screenshot: lastScreenshot,
-				screenshotSource: lastScreenshotSource,
-				failedStep: { code: mapped.code, message: mapped.message },
-			};
+	const completedUserIndices = new Set<number>();
+	for (const result of nativeResult.results) {
+		const wireStep = wireSteps[result.index];
+		if (!wireStep) continue;
+		if (wireStep.userIndex !== undefined) completedUserIndices.add(wireStep.userIndex);
+		const screenshot = normalizeScreenshot(result);
+		if (screenshot) {
+			lastScreenshot = screenshot;
+			lastScreenshotSource = result.screenshot;
+			if (wireStep.userIndex !== undefined) steps[wireStep.userIndex]!.screenshot = screenshot;
 		}
-		steps.push(stepDetails);
 	}
-	if (includeBatchScreenshot) {
-		lastScreenshotSource = await captureScreenshot(controller, deadline, signal);
-		lastScreenshot = normalizeScreenshot(lastScreenshotSource) ?? lastScreenshot;
+	if (!nativeResult.failureCode) {
+		throwIfAborted(signal);
+		assertComputerDeadline(deadline);
+		for (const step of steps) step.message = describeComputerSuccess(step);
+		return { steps, screenshot: lastScreenshot, screenshotSource: lastScreenshotSource };
 	}
-	return { steps, screenshot: lastScreenshot, screenshotSource: lastScreenshotSource };
+
+	const mapped = mapNativeBatchFailure(nativeResult.failureCode, nativeResult.failureMessage, hotkey);
+	const failedWireStep = wireSteps[nativeResult.failureIndex ?? nativeResult.results.length];
+	const primaryError =
+		nativeResult.primaryFailureCode === undefined
+			? undefined
+			: {
+					code: nativeResult.primaryFailureCode,
+					message: nativeResult.primaryFailureMessage ?? "Computer batch failed.",
+				};
+	if (failedWireStep?.userIndex !== undefined) {
+		const failed = steps[failedWireStep.userIndex]!;
+		failed.status = mapped.code === COMPUTER_DISABLED_CODE ? "disabled" : "error";
+		failed.code = mapped.code;
+		failed.message = mapped.message;
+	}
+	for (const index of completedUserIndices) {
+		const step = steps[index];
+		if (step && !step.message && step.status === "success") {
+			step.message = describeComputerSuccess(step);
+		}
+	}
+	const reportedIndices = new Set(completedUserIndices);
+	if (failedWireStep?.userIndex !== undefined) reportedIndices.add(failedWireStep.userIndex);
+	const reportedSteps = [...reportedIndices]
+		.sort((left, right) => left - right)
+		.map(index => steps[index]!)
+		.filter(Boolean);
+	return {
+		steps: reportedSteps,
+		screenshot: lastScreenshot,
+		screenshotSource: lastScreenshotSource,
+		failedStep: mapped,
+		primaryError,
+	};
 }
 
 function stepTimeoutFromParams(params: SingleComputerParams, batchTimeoutMs: number | undefined): number | undefined {
 	if (params.timeout === undefined) return batchTimeoutMs;
 	const timeoutSeconds = clampTimeout("computer", params.timeout);
-	return timeoutSeconds > 0 ? timeoutSeconds * 1000 : undefined;
+	const stepTimeoutMs = timeoutSeconds > 0 ? timeoutSeconds * 1000 : undefined;
+	if (stepTimeoutMs === undefined || batchTimeoutMs === undefined) return stepTimeoutMs ?? batchTimeoutMs;
+	return Math.min(stepTimeoutMs, batchTimeoutMs);
 }
 
 function detailsFromParams(params: ComputerParams): ComputerToolDetails {
@@ -821,6 +1093,9 @@ function mapComputerError(error: unknown, hotkey?: string): { code: string; mess
 			"The host needs screen-recording or accessibility permission. Ask the user to grant it.",
 		COMPUTER_DISABLED:
 			"The computer tool is disabled or unsupported. Do not retry without enabling it on Apple Silicon macOS.",
+		COMPUTER_SCREENSHOT_FAILED: "Capture failed. Check screen-recording permission and retry the screenshot.",
+		COMPUTER_CURSOR_RESTORE_FAILED:
+			"Input may have completed, but the cursor could not be restored. Stop and ask the user to inspect the desktop before retrying.",
 	};
 	const hint = recoveryHints[code];
 	const message = hint ? `${code}: ${reason} ${hint}` : `${code}: ${reason}`;
@@ -832,6 +1107,7 @@ interface ComputerAuditRecord {
 	action: ComputerActionName;
 	status: "success" | "error" | "disabled";
 	code?: string;
+	primaryError?: { code: string; message: string };
 	x?: number;
 	y?: number;
 	toX?: number;
@@ -853,6 +1129,7 @@ function auditRecordFromDetails(details: ComputerToolDetails): ComputerAuditReco
 		status: details.status,
 	};
 	if (details.code) record.code = details.code;
+	if (details.primaryError) record.primaryError = details.primaryError;
 	if (details.x !== undefined) record.x = details.x;
 	if (details.y !== undefined) record.y = details.y;
 	if (details.toX !== undefined) record.toX = details.toX;

package/src/tools/todo-write.ts

@@ -39,6 +39,7 @@ export interface TodoPhase {
 export interface TodoWriteToolDetails {
 	phases: TodoPhase[];
 	storage: "session" | "memory";
+	failureKind?: "payload_rejected" | "persistence";
 }
 
 // =============================================================================
@@ -520,6 +521,9 @@ function formatSummary(phases: TodoPhase[], errors: string[]): string {
 	}
 	return lines.join("\n");
 }
+function formatPayloadRejectedSummary(phases: TodoPhase[], errors: string[]): string {
+	return `${formatSummary(phases, errors)}\nTodo update was not applied.`;
+}
 
 // =============================================================================
 // Tool Class
@@ -547,26 +551,32 @@ export class TodoWriteTool implements AgentTool<typeof todoWriteSchema, TodoWrit
 		_context?: AgentToolContext,
 	): Promise<AgentToolResult<TodoWriteToolDetails>> {
 		const previousPhases = clonePhases(this.session.getTodoPhases?.() ?? []);
+		const storage = this.session.getSessionFile() ? "session" : "memory";
 		const missingTarget = params.ops.find(
 			entry => (entry.op === "done" || entry.op === "drop") && !entry.task && !entry.phase,
 		);
 		if (missingTarget) {
-			const storage = this.session.getSessionFile() ? "session" : "memory";
 			const errors = [`Missing task or phase for ${missingTarget.op} operation`];
 			return {
-				content: [{ type: "text", text: formatSummary(previousPhases, errors) }],
-				details: { phases: previousPhases, storage },
+				content: [{ type: "text", text: formatPayloadRejectedSummary(previousPhases, errors) }],
+				details: { phases: previousPhases, storage, failureKind: "payload_rejected" },
 				isError: true,
 			};
 		}
-		const { phases: updated, errors } = applyParams(previousPhases, params);
-		this.session.setTodoPhases?.(updated);
-		const storage = this.session.getSessionFile() ? "session" : "memory";
+		const { phases: updated, errors } = applyParams(clonePhases(previousPhases), params);
+		const rejected = errors.length > 0;
+		if (!rejected) this.session.setTodoPhases?.(updated);
+		const phases = rejected ? previousPhases : updated;
 
 		return {
-			content: [{ type: "text", text: formatSummary(updated, errors) }],
-			details: { phases: updated, storage },
-			isError: errors.length > 0 ? true : undefined,
+			content: [
+				{
+					type: "text",
+					text: rejected ? formatPayloadRejectedSummary(phases, errors) : formatSummary(phases, errors),
+				},
+			],
+			details: rejected ? { phases, storage, failureKind: "payload_rejected" } : { phases, storage },
+			isError: rejected || undefined,
 		};
 	}
 }
@@ -698,19 +708,27 @@ export const todoWriteToolRenderer = {
 	},
 
 	renderResult(
-		result: { content: Array<{ type: string; text?: string }>; details?: TodoWriteToolDetails },
+		result: {
+			content: Array<{ type: string; text?: string }>;
+			details?: TodoWriteToolDetails;
+			isError?: boolean;
+		},
 		options: RenderResultOptions,
 		uiTheme: Theme,
 		_args?: TodoWriteRenderArgs,
 	): Component {
 		const phases = (result.details?.phases ?? []).filter(phase => phase.tasks.length > 0);
 		const allTasks = phases.flatMap(phase => phase.tasks);
+		const fallback = result.content?.find(content => content.type === "text")?.text ?? "No todos";
+		if (result.isError) {
+			const header = renderStatusLine({ icon: "error", title: "Todo Write" }, uiTheme);
+			return new Text(`${header}\n${uiTheme.fg("error", fallback)}`, 0, 0);
+		}
 		const header = renderStatusLine(
 			{ icon: "success", title: "Todo Write", meta: [`${allTasks.length} tasks`] },
 			uiTheme,
 		);
 		if (allTasks.length === 0) {
-			const fallback = result.content?.find(content => content.type === "text")?.text ?? "No todos";
 			return new Text(`${header}\n${uiTheme.fg("dim", fallback)}`, 0, 0);
 		}