@agent-sh/computer-use-linux 0.7.13 → 0.7.15

package/README.md

@@ -78,7 +78,9 @@ and measure from its clipped screenshot crop origin. Divide preview `x` and
 GDK surface or widget-local coordinates; decorations and clipping can change
 the origin. A missing window target is rejected.
 
-Targeted `press_key`/`type_text` results append focused-element feedback from AT-SPI (role, name, editable) and warn when no editable element holds focus. Click/screenshot/input results warn when the target window or coordinate is partially or fully off-screen. `get_app_state` returns a compact readiness block by default; pass `verbose: true` for the full diagnostics report. It also reports `tree_scoped` (false when no app target narrowed the AT-SPI tree, with a warning in `message`) and `accessibility_tree_truncated` (true when the node, depth, or read budget stopped traversal with unread elements left).
+Targeted `press_key`/`type_text` results append focused-element feedback from AT-SPI (role, name, editable) and warn when a concrete non-terminal focus is not editable. Terminals accept keyboard input without an EditableText interface; re-observe the contents to confirm input. Click/screenshot/input results warn when the target window or coordinate is partially or fully off-screen. `get_app_state` returns a compact readiness block by default; pass `verbose: true` for the full diagnostics report. It also reports `tree_scoped` (false when no app target narrowed the AT-SPI tree, with a warning in `message`) and `accessibility_tree_truncated` (true when the node, depth, or read budget stopped traversal with unread elements left).
+
+On X11, text input sends LF and CRLF line endings as Return, with one newline per CRLF pair.
 
 On GNOME, a read-only keymap preflight rejects the whole string before input if a character is absent or requires a modifier level Mutter cannot synthesize. The background reader cannot establish the active layout group, so every configured group must support the text. This can reject text that works in one of several groups; use `set_value` on an editable field instead. The check requires `libxkbcommon.so.0` at runtime and stops after a two-second keymap read deadline. It does not cover held modifiers or a keymap changed concurrently with typing.
 
@@ -330,6 +332,16 @@ You can also search by capability:
 computer_use_linux_tools({ query: "observe a window and click a control" })
 ```
 
+Codemode scripts receive `{ text, images, isError }`. Each image is an MCP image
+block that can be passed directly to `image()`:
+
+```js
+const result = await tools.computer_use_linux_screenshot({});
+if (result.isError) throw new Error(result.text);
+text(result.text);
+for (const block of result.images) image(block);
+```
+
 No separate MCP adapter or manual MCP configuration is required. Pi starts one
 computer-use-linux process lazily on the first real tool call, reuses it for the
 session so accessibility snapshots remain valid, serializes desktop actions,

package/npm/README.md

@@ -88,6 +88,14 @@ When installed through Pi, the package supplies native, dynamically loaded
 configuration is required. Native tools require Pi 0.84.4 or newer; the
 standalone CLI wrapper retains Node.js 18 support.
 
+In Pi codemode, native tools return `{ text, images, isError }`. Pass image
+blocks directly to `image(result.images[0])` and check `isError` before using
+a result.
+
 If you already built or installed the binary yourself, set
 `COMPUTER_USE_LINUX_BIN=/path/to/computer-use-linux` to make the wrapper use
 that executable instead.
+
+X11 text input sends LF and CRLF line endings as Return. Focused terminals
+accept keyboard input even when AT-SPI does not expose EditableText. Focus
+feedback describes the target; re-observe its contents to confirm input.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-sh/computer-use-linux",
-  "version": "0.7.13",
+  "version": "0.7.15",
   "description": "Linux desktop-control MCP server: AT-SPI accessibility trees, Wayland/X11 input, screenshots, and compositor window targeting.",
   "license": "MIT",
   "type": "commonjs",

package/pi/extension/generated-tools.ts

@@ -7,7 +7,7 @@ export interface GeneratedMcpToolDefinition {
 	annotations: Record<string, unknown>;
 }
 
-export const GENERATED_SERVER_VERSION = "0.7.13";
+export const GENERATED_SERVER_VERSION = "0.7.15";
 export const GENERATED_TOOL_CATALOG_HASH = "666f4649084b7c1a240e6a0eacc01a55379b64a579355f50071211437e91229c";
 export const GENERATED_SHELL_TOOL_CATALOG_HASH = "554322e70886f5c1e1a3ad06869c5be91c140103f71350ad635160e128c1563d";
 export const GENERATED_MCP_TOOLS =

package/pi/extension/index.ts

@@ -483,6 +483,46 @@ function convertMcpResult(result: McpCallToolResult): PiContent[] {
 	return converted;
 }
 
+/**
+ * Machine-readable result for codemode scripts.
+ *
+ * Pi's codemode sandbox only passes `structuredContent` to nested calls from
+ * tools that declare `outputSchema`; otherwise scripts receive text only and
+ * image blocks are dropped. Mirror the `read` tool pattern so screenshots and
+ * other image results stay reachable via `image()`.
+ */
+const NativeToolOutputSchema = Type.Object({
+	text: Type.String({
+		description: "Joined text blocks of the converted MCP result.",
+	}),
+	images: Type.Array(
+		Type.Object({
+			type: Type.Literal("image"),
+			data: Type.String({ description: "Base64 image data." }),
+			mimeType: Type.String({ description: "Image MIME type." }),
+		}),
+		{ description: "Image blocks of the converted MCP result." },
+	),
+	isError: Type.Boolean({ description: "Whether the MCP tool reported a failure." }),
+});
+
+function toNativeToolStructuredContent(content: PiContent[], isError: boolean): {
+	text: string;
+	images: Array<{ type: "image"; data: string; mimeType: string }>;
+	isError: boolean;
+} {
+	const texts: string[] = [];
+	const images: Array<{ type: "image"; data: string; mimeType: string }> = [];
+	for (const block of content) {
+		if (block.type === "text") {
+			texts.push(block.text);
+		} else if (block.type === "image") {
+			images.push({ type: "image", data: block.data, mimeType: block.mimeType });
+		}
+	}
+	return { text: texts.join("\n"), images, isError };
+}
+
 function legacyConfigPath(): string {
 	const agentDir =
 		process.env.PI_CODING_AGENT_DIR?.trim() || join(homedir(), ".pi", "agent");
@@ -585,6 +625,7 @@ export function createComputerUseLinuxExtension(
 				label: `Computer Use: ${tool.name}`,
 				description: nativeToolDescription(tool),
 				parameters: toolParameters(tool.inputSchema),
+				outputSchema: NativeToolOutputSchema,
 				executionMode: "sequential",
 				async execute(_toolCallId, params, signal) {
 					const result = await getClient().callTool(
@@ -592,13 +633,15 @@ export function createComputerUseLinuxExtension(
 						params as Record<string, unknown>,
 						signal,
 					);
+					const content = convertMcpResult(result);
 					return {
-						content: convertMcpResult(result),
+						content,
 						details: {
 							computerUseLinux: true,
 							mcpIsError: result.isError === true,
 							tool: tool.name,
 						},
+						structuredContent: toNativeToolStructuredContent(content, result.isError === true),
 					};
 				},
 			});

package/skills/computer-use-linux/references/pi-setup.md

@@ -59,6 +59,16 @@ Pi does not start the desktop process when the loader runs. The first real tool
 call starts one computer-use-linux process, and the package reuses it for the
 session so `get_app_state` element indices and portal sessions remain valid.
 
+Codemode scripts receive `{ text, images, isError }`. Each image is an MCP image
+block that can be passed directly to `image()`:
+
+```js
+const result = await tools.computer_use_linux_screenshot({});
+if (result.isError) throw new Error(result.text);
+text(result.text);
+for (const block of result.images) image(block);
+```
+
 ## Safe operating loop
 
 1. Enable `get_app_state`, `list_windows`, and any likely action tools.