@agent-sh/computer-use-linux 0.7.13 → 0.7.14

package/README.md

@@ -330,6 +330,16 @@ You can also search by capability:
 computer_use_linux_tools({ query: "observe a window and click a control" })
 ```
 
+Codemode scripts receive `{ text, images, isError }`. Each image is an MCP image
+block that can be passed directly to `image()`:
+
+```js
+const result = await tools.computer_use_linux_screenshot({});
+if (result.isError) throw new Error(result.text);
+text(result.text);
+for (const block of result.images) image(block);
+```
+
 No separate MCP adapter or manual MCP configuration is required. Pi starts one
 computer-use-linux process lazily on the first real tool call, reuses it for the
 session so accessibility snapshots remain valid, serializes desktop actions,

package/npm/README.md

@@ -88,6 +88,10 @@ When installed through Pi, the package supplies native, dynamically loaded
 configuration is required. Native tools require Pi 0.84.4 or newer; the
 standalone CLI wrapper retains Node.js 18 support.
 
+In Pi codemode, native tools return `{ text, images, isError }`. Pass image
+blocks directly to `image(result.images[0])` and check `isError` before using
+a result.
+
 If you already built or installed the binary yourself, set
 `COMPUTER_USE_LINUX_BIN=/path/to/computer-use-linux` to make the wrapper use
 that executable instead.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-sh/computer-use-linux",
-  "version": "0.7.13",
+  "version": "0.7.14",
   "description": "Linux desktop-control MCP server: AT-SPI accessibility trees, Wayland/X11 input, screenshots, and compositor window targeting.",
   "license": "MIT",
   "type": "commonjs",

package/pi/extension/generated-tools.ts

@@ -7,7 +7,7 @@ export interface GeneratedMcpToolDefinition {
 	annotations: Record<string, unknown>;
 }
 
-export const GENERATED_SERVER_VERSION = "0.7.13";
+export const GENERATED_SERVER_VERSION = "0.7.14";
 export const GENERATED_TOOL_CATALOG_HASH = "666f4649084b7c1a240e6a0eacc01a55379b64a579355f50071211437e91229c";
 export const GENERATED_SHELL_TOOL_CATALOG_HASH = "554322e70886f5c1e1a3ad06869c5be91c140103f71350ad635160e128c1563d";
 export const GENERATED_MCP_TOOLS =

package/pi/extension/index.ts

@@ -483,6 +483,46 @@ function convertMcpResult(result: McpCallToolResult): PiContent[] {
 	return converted;
 }
 
+/**
+ * Machine-readable result for codemode scripts.
+ *
+ * Pi's codemode sandbox only passes `structuredContent` to nested calls from
+ * tools that declare `outputSchema`; otherwise scripts receive text only and
+ * image blocks are dropped. Mirror the `read` tool pattern so screenshots and
+ * other image results stay reachable via `image()`.
+ */
+const NativeToolOutputSchema = Type.Object({
+	text: Type.String({
+		description: "Joined text blocks of the converted MCP result.",
+	}),
+	images: Type.Array(
+		Type.Object({
+			type: Type.Literal("image"),
+			data: Type.String({ description: "Base64 image data." }),
+			mimeType: Type.String({ description: "Image MIME type." }),
+		}),
+		{ description: "Image blocks of the converted MCP result." },
+	),
+	isError: Type.Boolean({ description: "Whether the MCP tool reported a failure." }),
+});
+
+function toNativeToolStructuredContent(content: PiContent[], isError: boolean): {
+	text: string;
+	images: Array<{ type: "image"; data: string; mimeType: string }>;
+	isError: boolean;
+} {
+	const texts: string[] = [];
+	const images: Array<{ type: "image"; data: string; mimeType: string }> = [];
+	for (const block of content) {
+		if (block.type === "text") {
+			texts.push(block.text);
+		} else if (block.type === "image") {
+			images.push({ type: "image", data: block.data, mimeType: block.mimeType });
+		}
+	}
+	return { text: texts.join("\n"), images, isError };
+}
+
 function legacyConfigPath(): string {
 	const agentDir =
 		process.env.PI_CODING_AGENT_DIR?.trim() || join(homedir(), ".pi", "agent");
@@ -585,6 +625,7 @@ export function createComputerUseLinuxExtension(
 				label: `Computer Use: ${tool.name}`,
 				description: nativeToolDescription(tool),
 				parameters: toolParameters(tool.inputSchema),
+				outputSchema: NativeToolOutputSchema,
 				executionMode: "sequential",
 				async execute(_toolCallId, params, signal) {
 					const result = await getClient().callTool(
@@ -592,13 +633,15 @@ export function createComputerUseLinuxExtension(
 						params as Record<string, unknown>,
 						signal,
 					);
+					const content = convertMcpResult(result);
 					return {
-						content: convertMcpResult(result),
+						content,
 						details: {
 							computerUseLinux: true,
 							mcpIsError: result.isError === true,
 							tool: tool.name,
 						},
+						structuredContent: toNativeToolStructuredContent(content, result.isError === true),
 					};
 				},
 			});

package/skills/computer-use-linux/references/pi-setup.md

@@ -59,6 +59,16 @@ Pi does not start the desktop process when the loader runs. The first real tool
 call starts one computer-use-linux process, and the package reuses it for the
 session so `get_app_state` element indices and portal sessions remain valid.
 
+Codemode scripts receive `{ text, images, isError }`. Each image is an MCP image
+block that can be passed directly to `image()`:
+
+```js
+const result = await tools.computer_use_linux_screenshot({});
+if (result.isError) throw new Error(result.text);
+text(result.text);
+for (const block of result.images) image(block);
+```
+
 ## Safe operating loop
 
 1. Enable `get_app_state`, `list_windows`, and any likely action tools.