@agent-sh/computer-use-linux 0.7.42 → 0.7.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -87
- package/npm/README.md +27 -17
- package/package.json +7 -4
- package/pi/extension/generated-tools.ts +119 -27
- package/pi/extension/index.ts +245 -97
- package/skills/computer-use-linux/SKILL.md +27 -17
- package/skills/computer-use-linux/references/hermes-setup.md +6 -3
- package/skills/computer-use-linux/references/pi-setup.md +18 -8
package/pi/extension/index.ts
CHANGED
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
readFileSync,
|
|
21
21
|
statSync,
|
|
22
22
|
} from "node:fs";
|
|
23
|
+
import { spawn } from "node:child_process";
|
|
23
24
|
import { createRequire } from "node:module";
|
|
24
25
|
import { homedir } from "node:os";
|
|
25
26
|
import { delimiter, isAbsolute, join } from "node:path";
|
|
@@ -95,8 +96,11 @@ interface BinaryLaunch {
|
|
|
95
96
|
}
|
|
96
97
|
|
|
97
98
|
interface ExtensionDependencies {
|
|
99
|
+
/** Replaces the whole lookup, the first-use download included. */
|
|
98
100
|
findBinary?: () => BinaryLaunch | null;
|
|
99
101
|
loadClientModule?: () => NativeMcpClientModule;
|
|
102
|
+
/** Directory holding npm/bin and npm/install.js. Defaults to this package. */
|
|
103
|
+
packageRoot?: string;
|
|
100
104
|
}
|
|
101
105
|
|
|
102
106
|
const ORIGINAL_TOOL_NAMES = AVAILABLE_MCP_TOOLS.map((tool) => tool.name);
|
|
@@ -191,57 +195,72 @@ export function findExecutableOnPath(
|
|
|
191
195
|
return null;
|
|
192
196
|
}
|
|
193
197
|
|
|
198
|
+
/**
|
|
199
|
+
* Variables the server child receives, plus every LC_ variable. Everything
|
|
200
|
+
* else in Pi's environment stays out of the desktop process.
|
|
201
|
+
* scripts/check_env_forwarding.py fails CI when the server reads a variable
|
|
202
|
+
* missing here. COMPUTER_USE_LINUX_NOTIFY_ON_COMPLETE is left out: it adds a
|
|
203
|
+
* tool, and the client refuses a catalog that differs from the generated one.
|
|
204
|
+
*/
|
|
205
|
+
const FORWARDED_ENVIRONMENT = [
|
|
206
|
+
"PATH",
|
|
207
|
+
"HOME",
|
|
208
|
+
"USER",
|
|
209
|
+
"LOGNAME",
|
|
210
|
+
"SHELL",
|
|
211
|
+
"LANG",
|
|
212
|
+
"LANGUAGE",
|
|
213
|
+
"TERM",
|
|
214
|
+
"TMPDIR",
|
|
215
|
+
"DISPLAY",
|
|
216
|
+
"WAYLAND_DISPLAY",
|
|
217
|
+
"XAUTHORITY",
|
|
218
|
+
"DBUS_SESSION_BUS_ADDRESS",
|
|
219
|
+
"XDG_RUNTIME_DIR",
|
|
220
|
+
"XDG_CURRENT_DESKTOP",
|
|
221
|
+
"XDG_SESSION_TYPE",
|
|
222
|
+
"XDG_SESSION_DESKTOP",
|
|
223
|
+
"XDG_DATA_DIRS",
|
|
224
|
+
"XDG_CONFIG_HOME",
|
|
225
|
+
"XDG_CACHE_HOME",
|
|
226
|
+
"XDG_STATE_HOME",
|
|
227
|
+
"DESKTOP_SESSION",
|
|
228
|
+
"GDMSESSION",
|
|
229
|
+
"HYPRLAND_INSTANCE_SIGNATURE",
|
|
230
|
+
"NIRI_SOCKET",
|
|
231
|
+
"I3SOCK",
|
|
232
|
+
"SWAYSOCK",
|
|
233
|
+
"YDOTOOL_SOCKET",
|
|
234
|
+
"GSETTINGS_SCHEMA_DIR",
|
|
235
|
+
"GIO_EXTRA_MODULES",
|
|
236
|
+
"GI_TYPELIB_PATH",
|
|
237
|
+
"LD_LIBRARY_PATH",
|
|
238
|
+
"NIX_LD",
|
|
239
|
+
"NIX_LD_LIBRARY_PATH",
|
|
240
|
+
"RUST_LOG",
|
|
241
|
+
"COMPUTER_USE_LINUX_AGENT_NAME",
|
|
242
|
+
"COMPUTER_USE_LINUX_COSMIC_HELPER",
|
|
243
|
+
"COMPUTER_USE_LINUX_ENABLE_SHELL",
|
|
244
|
+
"COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD",
|
|
245
|
+
"COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER",
|
|
246
|
+
"COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD",
|
|
247
|
+
"COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD",
|
|
248
|
+
"COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER",
|
|
249
|
+
"COMPUTER_USE_LINUX_HYPRLAND_BACKGROUND",
|
|
250
|
+
"COMPUTER_USE_LINUX_INDICATOR",
|
|
251
|
+
"COMPUTER_USE_LINUX_INDICATOR_BIN",
|
|
252
|
+
"COMPUTER_USE_LINUX_INDICATOR_HIDE_TEXT",
|
|
253
|
+
"COMPUTER_USE_LINUX_PERSIST_REMOTE_DESKTOP",
|
|
254
|
+
"COMPUTER_USE_LINUX_PORTAL_SCROLL_INVERT",
|
|
255
|
+
"COMPUTER_USE_LINUX_SCREENSHOT_BACKEND",
|
|
256
|
+
"COMPUTER_USE_LINUX_XDOTOOL_TYPE_DELAY_MS",
|
|
257
|
+
"CU_DISABLE_ABS_POINTER",
|
|
258
|
+
"PIXELFLUX_CU",
|
|
259
|
+
"PIXELFLUX_CU_TOKEN",
|
|
260
|
+
] as const;
|
|
261
|
+
|
|
194
262
|
function runtimeEnvironment(): Record<string, string> {
|
|
195
|
-
const allowed = new Set(
|
|
196
|
-
"PATH",
|
|
197
|
-
"HOME",
|
|
198
|
-
"USER",
|
|
199
|
-
"LOGNAME",
|
|
200
|
-
"SHELL",
|
|
201
|
-
"LANG",
|
|
202
|
-
"LANGUAGE",
|
|
203
|
-
"TERM",
|
|
204
|
-
"TMPDIR",
|
|
205
|
-
"DISPLAY",
|
|
206
|
-
"WAYLAND_DISPLAY",
|
|
207
|
-
"XAUTHORITY",
|
|
208
|
-
"DBUS_SESSION_BUS_ADDRESS",
|
|
209
|
-
"XDG_RUNTIME_DIR",
|
|
210
|
-
"XDG_CURRENT_DESKTOP",
|
|
211
|
-
"XDG_SESSION_TYPE",
|
|
212
|
-
"XDG_SESSION_DESKTOP",
|
|
213
|
-
"XDG_DATA_DIRS",
|
|
214
|
-
"XDG_CONFIG_HOME",
|
|
215
|
-
"XDG_CACHE_HOME",
|
|
216
|
-
"XDG_STATE_HOME",
|
|
217
|
-
"DESKTOP_SESSION",
|
|
218
|
-
"GDMSESSION",
|
|
219
|
-
"HYPRLAND_INSTANCE_SIGNATURE",
|
|
220
|
-
"I3SOCK",
|
|
221
|
-
"SWAYSOCK",
|
|
222
|
-
"YDOTOOL_SOCKET",
|
|
223
|
-
"GSETTINGS_SCHEMA_DIR",
|
|
224
|
-
"GIO_EXTRA_MODULES",
|
|
225
|
-
"GI_TYPELIB_PATH",
|
|
226
|
-
"LD_LIBRARY_PATH",
|
|
227
|
-
"NIX_LD",
|
|
228
|
-
"NIX_LD_LIBRARY_PATH",
|
|
229
|
-
"RUST_LOG",
|
|
230
|
-
"COMPUTER_USE_LINUX_COSMIC_HELPER",
|
|
231
|
-
"COMPUTER_USE_LINUX_AGENT_NAME",
|
|
232
|
-
"COMPUTER_USE_LINUX_INDICATOR",
|
|
233
|
-
"COMPUTER_USE_LINUX_INDICATOR_BIN",
|
|
234
|
-
"COMPUTER_USE_LINUX_INDICATOR_HIDE_TEXT",
|
|
235
|
-
"COMPUTER_USE_LINUX_ENABLE_SHELL",
|
|
236
|
-
"COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD",
|
|
237
|
-
"COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER",
|
|
238
|
-
"COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD",
|
|
239
|
-
"COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD",
|
|
240
|
-
"COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER",
|
|
241
|
-
"COMPUTER_USE_LINUX_PORTAL_SCROLL_INVERT",
|
|
242
|
-
"COMPUTER_USE_LINUX_SCREENSHOT_BACKEND",
|
|
243
|
-
"CU_DISABLE_ABS_POINTER",
|
|
244
|
-
]);
|
|
263
|
+
const allowed = new Set<string>(FORWARDED_ENVIRONMENT);
|
|
245
264
|
const env = Object.fromEntries(
|
|
246
265
|
Object.entries(process.env).filter(
|
|
247
266
|
(entry): entry is [string, string] =>
|
|
@@ -254,7 +273,16 @@ function runtimeEnvironment(): Record<string, string> {
|
|
|
254
273
|
return env;
|
|
255
274
|
}
|
|
256
275
|
|
|
257
|
-
|
|
276
|
+
/** Directory holding npm/bin and npm/install.js. */
|
|
277
|
+
const PACKAGE_ROOT = join(__dirname, "..", "..");
|
|
278
|
+
const INSTALL_OUTPUT_CHARS = 4000;
|
|
279
|
+
|
|
280
|
+
function installerPath(packageRoot: string): string {
|
|
281
|
+
return join(packageRoot, "npm", "install.js");
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** COMPUTER_USE_LINUX_BIN, then the package's own binary. */
|
|
285
|
+
function localLaunch(packageRoot: string): BinaryLaunch | null {
|
|
258
286
|
const env = runtimeEnvironment();
|
|
259
287
|
const override = process.env.COMPUTER_USE_LINUX_BIN?.trim();
|
|
260
288
|
// Relative overrides would be resolved from Pi's potentially untrusted
|
|
@@ -263,50 +291,93 @@ function defaultFindBinary(): BinaryLaunch | null {
|
|
|
263
291
|
return { binaryPath: override, env };
|
|
264
292
|
}
|
|
265
293
|
|
|
294
|
+
const binDir = join(packageRoot, "npm", "bin");
|
|
266
295
|
const bundledBinary = join(
|
|
267
|
-
|
|
268
|
-
"..",
|
|
269
|
-
"..",
|
|
270
|
-
"npm",
|
|
271
|
-
"bin",
|
|
296
|
+
binDir,
|
|
272
297
|
`computer-use-linux-${process.platform}-${process.arch}`,
|
|
273
298
|
);
|
|
274
|
-
if (executable(bundledBinary))
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
);
|
|
283
|
-
if (!env.COMPUTER_USE_LINUX_COSMIC_HELPER && executable(cosmicHelper)) {
|
|
284
|
-
env.COMPUTER_USE_LINUX_COSMIC_HELPER = cosmicHelper;
|
|
285
|
-
}
|
|
286
|
-
const indicator = join(
|
|
287
|
-
__dirname,
|
|
288
|
-
"..",
|
|
289
|
-
"..",
|
|
290
|
-
"npm",
|
|
291
|
-
"bin",
|
|
292
|
-
"computer-use-linux-indicator",
|
|
293
|
-
);
|
|
294
|
-
if (!env.COMPUTER_USE_LINUX_INDICATOR_BIN && executable(indicator)) {
|
|
295
|
-
env.COMPUTER_USE_LINUX_INDICATOR_BIN = indicator;
|
|
296
|
-
}
|
|
297
|
-
return { binaryPath: bundledBinary, env };
|
|
299
|
+
if (!executable(bundledBinary)) return null;
|
|
300
|
+
const cosmicHelper = join(binDir, "computer-use-linux-cosmic");
|
|
301
|
+
if (!env.COMPUTER_USE_LINUX_COSMIC_HELPER && executable(cosmicHelper)) {
|
|
302
|
+
env.COMPUTER_USE_LINUX_COSMIC_HELPER = cosmicHelper;
|
|
303
|
+
}
|
|
304
|
+
const indicator = join(binDir, "computer-use-linux-indicator");
|
|
305
|
+
if (!env.COMPUTER_USE_LINUX_INDICATOR_BIN && executable(indicator)) {
|
|
306
|
+
env.COMPUTER_USE_LINUX_INDICATOR_BIN = indicator;
|
|
298
307
|
}
|
|
308
|
+
return { binaryPath: bundledBinary, env };
|
|
309
|
+
}
|
|
299
310
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
311
|
+
/**
|
|
312
|
+
* A global npm or cargo install. The MCP client still checks the server
|
|
313
|
+
* version and tool catalog, so a mismatched install fails with that reason.
|
|
314
|
+
*/
|
|
315
|
+
function pathLaunch(): BinaryLaunch | null {
|
|
304
316
|
const onPath = findExecutableOnPath(process.env.PATH, "computer-use-linux");
|
|
305
|
-
|
|
306
|
-
|
|
317
|
+
return onPath ? { binaryPath: onPath, env: runtimeEnvironment() } : null;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Runs npm/install.js, the first-use download the CLI wrapper also runs.
|
|
322
|
+
* npm 12 skips install scripts it was not told to allow, so `pi install` can
|
|
323
|
+
* stage the package without its binary. The installer checks each download's
|
|
324
|
+
* sha256 and moves the main binary into place last. Its output is kept for
|
|
325
|
+
* the error and never reaches Pi's terminal.
|
|
326
|
+
*/
|
|
327
|
+
function runInstaller(packageRoot: string): Promise<void> {
|
|
328
|
+
const installer = installerPath(packageRoot);
|
|
329
|
+
// Under Bun, execPath is the Bun or Pi executable, not node.
|
|
330
|
+
const node = process.versions.bun
|
|
331
|
+
? findExecutableOnPath(process.env.PATH, "node")
|
|
332
|
+
: process.execPath;
|
|
333
|
+
if (!node) {
|
|
334
|
+
return Promise.reject(
|
|
335
|
+
new Error(`node was not found on PATH to run ${installer}`),
|
|
336
|
+
);
|
|
307
337
|
}
|
|
338
|
+
return new Promise((resolve, reject) => {
|
|
339
|
+
let output = "";
|
|
340
|
+
const keep = (chunk: Buffer) => {
|
|
341
|
+
output = (output + chunk.toString()).slice(-INSTALL_OUTPUT_CHARS);
|
|
342
|
+
};
|
|
343
|
+
const child = spawn(node, [installer], {
|
|
344
|
+
cwd: packageRoot,
|
|
345
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
346
|
+
});
|
|
347
|
+
child.stdout.on("data", keep);
|
|
348
|
+
child.stderr.on("data", keep);
|
|
349
|
+
child.on("error", reject);
|
|
350
|
+
child.on("close", (code, signal) => {
|
|
351
|
+
if (code === 0) {
|
|
352
|
+
resolve();
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
reject(
|
|
356
|
+
new Error(
|
|
357
|
+
`${installer} exited with ${signal ?? `status ${code}`}: ${output.trim()}`,
|
|
358
|
+
),
|
|
359
|
+
);
|
|
360
|
+
});
|
|
361
|
+
});
|
|
362
|
+
}
|
|
308
363
|
|
|
309
|
-
|
|
364
|
+
/** Settles with `promise`, or rejects with the signal's reason once it aborts. */
|
|
365
|
+
function untilAborted<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
366
|
+
if (!signal) return promise;
|
|
367
|
+
return new Promise((resolve, reject) => {
|
|
368
|
+
const onAbort = () => reject(signal.reason);
|
|
369
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
370
|
+
promise.then(
|
|
371
|
+
(value) => {
|
|
372
|
+
signal.removeEventListener("abort", onAbort);
|
|
373
|
+
resolve(value);
|
|
374
|
+
},
|
|
375
|
+
(error) => {
|
|
376
|
+
signal.removeEventListener("abort", onAbort);
|
|
377
|
+
reject(error);
|
|
378
|
+
},
|
|
379
|
+
);
|
|
380
|
+
});
|
|
310
381
|
}
|
|
311
382
|
|
|
312
383
|
function defaultLoadClientModule(): NativeMcpClientModule {
|
|
@@ -581,7 +652,12 @@ function restoredNativeTools(ctx: ExtensionContext): string[] {
|
|
|
581
652
|
export function createComputerUseLinuxExtension(
|
|
582
653
|
dependencies: ExtensionDependencies = {},
|
|
583
654
|
) {
|
|
584
|
-
const
|
|
655
|
+
const packageRoot = dependencies.packageRoot ?? PACKAGE_ROOT;
|
|
656
|
+
const findLocal =
|
|
657
|
+
dependencies.findBinary ?? (() => localLaunch(packageRoot));
|
|
658
|
+
const findOnPath = dependencies.findBinary ? () => null : pathLaunch;
|
|
659
|
+
const canInstall = () =>
|
|
660
|
+
!dependencies.findBinary && existsSync(installerPath(packageRoot));
|
|
585
661
|
const loadClientModule =
|
|
586
662
|
dependencies.loadClientModule ?? defaultLoadClientModule;
|
|
587
663
|
|
|
@@ -589,21 +665,64 @@ export function createComputerUseLinuxExtension(
|
|
|
589
665
|
let launch: BinaryLaunch | undefined;
|
|
590
666
|
let client: NativeMcpClient | undefined;
|
|
591
667
|
let enabledNativeTools = new Set<string>();
|
|
592
|
-
|
|
593
|
-
|
|
668
|
+
let install: Promise<string | undefined> | undefined;
|
|
669
|
+
let installFailure: string | undefined;
|
|
670
|
+
// True while the launch is a PATH binary taken because the package
|
|
671
|
+
// binary could not be downloaded. It may not match this extension.
|
|
672
|
+
let launchIsFallback = false;
|
|
673
|
+
// Bumped by session_shutdown, so a download that outlives the session
|
|
674
|
+
// never starts a server nothing would close.
|
|
675
|
+
let session = 0;
|
|
676
|
+
|
|
677
|
+
// The package's binary comes before PATH, as after an install whose
|
|
678
|
+
// script ran: it matches this extension's catalog.
|
|
679
|
+
const resolveLaunch = async () => {
|
|
594
680
|
if (launch) return launch;
|
|
595
|
-
|
|
681
|
+
let resolved = findLocal();
|
|
682
|
+
if (!resolved && canInstall()) {
|
|
683
|
+
install ??= runInstaller(packageRoot).then(
|
|
684
|
+
() => undefined,
|
|
685
|
+
(error: Error) => error.message,
|
|
686
|
+
);
|
|
687
|
+
const pending = install;
|
|
688
|
+
installFailure = await pending;
|
|
689
|
+
if (install === pending) install = undefined;
|
|
690
|
+
resolved = findLocal();
|
|
691
|
+
}
|
|
692
|
+
launchIsFallback = false;
|
|
693
|
+
if (!resolved) {
|
|
694
|
+
resolved = findOnPath();
|
|
695
|
+
launchIsFallback = resolved !== null && installFailure !== undefined;
|
|
696
|
+
}
|
|
596
697
|
if (resolved) launch = resolved;
|
|
597
698
|
return resolved;
|
|
598
699
|
};
|
|
599
700
|
|
|
600
|
-
const
|
|
701
|
+
const dropClient = async () => {
|
|
702
|
+
const current = client;
|
|
703
|
+
client = undefined;
|
|
704
|
+
launch = undefined;
|
|
705
|
+
launchIsFallback = false;
|
|
706
|
+
await current?.close();
|
|
707
|
+
};
|
|
708
|
+
|
|
709
|
+
const getClient = async () => {
|
|
710
|
+
const started = session;
|
|
711
|
+
// The package binary appeared since the fallback was chosen.
|
|
712
|
+
if (launchIsFallback && findLocal()) await dropClient();
|
|
713
|
+
if (client) return client;
|
|
714
|
+
const resolved = await resolveLaunch();
|
|
715
|
+
if (session !== started) {
|
|
716
|
+
throw new Error("Pi session ended while computer-use-linux was being installed");
|
|
717
|
+
}
|
|
601
718
|
if (client) return client;
|
|
602
|
-
const resolved = resolveLaunch();
|
|
603
719
|
if (!resolved) {
|
|
720
|
+
const failure = installFailure
|
|
721
|
+
? ` Downloading it failed: ${installFailure}`
|
|
722
|
+
: "";
|
|
604
723
|
throw new Error(
|
|
605
|
-
|
|
606
|
-
`${PACKAGE_NAME}, install computer-use-linux on PATH, or set COMPUTER_USE_LINUX_BIN.`,
|
|
724
|
+
`computer-use-linux binary was not found.${failure} Reinstall ` +
|
|
725
|
+
`${PACKAGE_NAME} with its install script allowed, install computer-use-linux on PATH, or set COMPUTER_USE_LINUX_BIN.`,
|
|
607
726
|
);
|
|
608
727
|
}
|
|
609
728
|
const { ComputerUseMcpClient } = loadClientModule();
|
|
@@ -620,6 +739,32 @@ export function createComputerUseLinuxExtension(
|
|
|
620
739
|
return client;
|
|
621
740
|
};
|
|
622
741
|
|
|
742
|
+
const callTool = async (
|
|
743
|
+
name: string,
|
|
744
|
+
args: Record<string, unknown>,
|
|
745
|
+
signal: AbortSignal | undefined,
|
|
746
|
+
) => {
|
|
747
|
+
// A cancelled call never starts a download. A call cancelled while
|
|
748
|
+
// one runs stops waiting; the download goes on for the next call.
|
|
749
|
+
signal?.throwIfAborted();
|
|
750
|
+
const active = await untilAborted(getClient(), signal);
|
|
751
|
+
try {
|
|
752
|
+
return await active.callTool(name, args, signal);
|
|
753
|
+
} catch (error) {
|
|
754
|
+
// A PATH binary taken after a failed download can be the wrong
|
|
755
|
+
// version. Drop it so the next call tries the download again.
|
|
756
|
+
if (signal?.aborted || !launchIsFallback || client !== active) throw error;
|
|
757
|
+
const failure = installFailure;
|
|
758
|
+
await dropClient();
|
|
759
|
+
throw new Error(
|
|
760
|
+
`${(error instanceof Error ? error.message : String(error)).replace(/\.$/, "")}. ` +
|
|
761
|
+
`This computer-use-linux came from PATH because downloading the package binary failed: ${failure}. ` +
|
|
762
|
+
"The next call tries the download again.",
|
|
763
|
+
{ cause: error },
|
|
764
|
+
);
|
|
765
|
+
}
|
|
766
|
+
};
|
|
767
|
+
|
|
623
768
|
for (const tool of AVAILABLE_MCP_TOOLS) {
|
|
624
769
|
const nativeName = `${TOOL_PREFIX}${tool.name}`;
|
|
625
770
|
pi.registerTool({
|
|
@@ -630,7 +775,7 @@ export function createComputerUseLinuxExtension(
|
|
|
630
775
|
outputSchema: NativeToolOutputSchema,
|
|
631
776
|
executionMode: "sequential",
|
|
632
777
|
async execute(_toolCallId, params, signal) {
|
|
633
|
-
const result = await
|
|
778
|
+
const result = await callTool(
|
|
634
779
|
tool.name,
|
|
635
780
|
params as Record<string, unknown>,
|
|
636
781
|
signal,
|
|
@@ -722,7 +867,9 @@ export function createComputerUseLinuxExtension(
|
|
|
722
867
|
}
|
|
723
868
|
pi.setActiveTools(active);
|
|
724
869
|
|
|
725
|
-
|
|
870
|
+
// Nothing is downloaded here; a missing package binary is fetched on
|
|
871
|
+
// the first tool call.
|
|
872
|
+
if (!findLocal() && !canInstall() && !findOnPath() && ctx.hasUI) {
|
|
726
873
|
ctx.ui.notify(
|
|
727
874
|
`${PACKAGE_NAME}: binary not found; reinstall the package, install computer-use-linux on PATH, or set COMPUTER_USE_LINUX_BIN.`,
|
|
728
875
|
"warning",
|
|
@@ -746,6 +893,7 @@ export function createComputerUseLinuxExtension(
|
|
|
746
893
|
});
|
|
747
894
|
|
|
748
895
|
pi.on("session_shutdown", async () => {
|
|
896
|
+
session += 1;
|
|
749
897
|
const current = client;
|
|
750
898
|
client = undefined;
|
|
751
899
|
await current?.close();
|
|
@@ -11,7 +11,7 @@ compatibility: "Native Pi tools require Pi 0.84.4+ and Node.js 22.19+; the stand
|
|
|
11
11
|
|
|
12
12
|
Use `computer-use-linux` when an agent needs to observe or operate a local Linux desktop: inspect the accessibility tree, list/focus windows, take screenshots, click, scroll, type, press keys, or invoke AT-SPI actions.
|
|
13
13
|
|
|
14
|
-
On labwc, window targeting uses the Selkies IPC patch's `labwc.sock` when it exists. Stock labwc and other wlroots compositors that advertise wlr-foreign-toplevel-management use that protocol instead, which reports no pids or bounds. Activation requires a unique app-id/title match. Window IDs include the title and must be refreshed after it changes. Ambiguous windows are rejected. Window bounds and relative coordinate targeting are unavailable until output scaling can be verified.
|
|
14
|
+
On labwc, window targeting uses the Selkies IPC patch's `labwc.sock` when it exists. Stock labwc and other wlroots compositors that advertise wlr-foreign-toplevel-management use that protocol instead, which reports no pids or bounds. Activation requires a unique app-id/title match. Window IDs include the title and must be refreshed after it changes. Ambiguous windows are rejected. Window bounds and relative coordinate targeting are unavailable until output scaling can be verified. Window-targeted screenshots, including `screenshot_after: "window"`, fail for the same reason; capture the full screen instead.
|
|
15
15
|
|
|
16
16
|
Native X11 clicks, scrolls and drags use xdotool/XTEST on `DISPLAY`, ahead of uinput and portal input. Install xdotool for this route. Missing or failed commands return an error without a system-wide fallback. Drags send paced intermediate motion and always attempt release after a failed press or motion; verify the drop target afterward.
|
|
17
17
|
|
|
@@ -81,9 +81,11 @@ computer-use-linux setup-window-targeting
|
|
|
81
81
|
computer-use-linux doctor | jq .readiness
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
-
If `doctor` selects ydotool as the input backend, also enable its per-user daemon
|
|
84
|
+
If `doctor` selects ydotool as the input backend, also enable its per-user daemon. `./install.sh` creates `ydotoold.service`; Ubuntu's ydotool package ships `ydotool.service` instead (`systemctl --user enable --now ydotool`). Direct uinput, X11 xdotool, and RemoteDesktop portal input do not require `ydotoold`, but a Wayland scroll at a point needs ydotool for the wheel, or the portal.
|
|
85
85
|
|
|
86
|
-
On niri,
|
|
86
|
+
On niri, the server reads `NIRI_SOCKET` from its parent processes or the systemd
|
|
87
|
+
user environment when the host does not pass it; without it, discovery needs an
|
|
88
|
+
unambiguous socket matching `WAYLAND_DISPLAY`. Window listing and
|
|
87
89
|
exact focus use `niri msg` with a direct IPC fallback. Missing positions or
|
|
88
90
|
unknown/mixed output scaling leave window-relative coordinates unavailable;
|
|
89
91
|
focus the target and inspect a fresh full-screen screenshot before using
|
|
@@ -139,15 +141,15 @@ Pi native tools skip this MCP `command` config; see [Pi setup](references/pi-set
|
|
|
139
141
|
## Procedure
|
|
140
142
|
|
|
141
143
|
1. In Pi, call `computer_use_linux_tools` with the exact tools or capability you need. Enabled tools use the `computer_use_linux_<name>` prefix, appear starting on the next model turn, and remain active for the session.
|
|
142
|
-
2. Begin every desktop-control turn with `get_app_state`, scoped to the app you are working in: pass `app_name_or_bundle_identifier` or a window target (`window_id`, `pid`, `app_id`, `wm_class`, `title`). An `app_id` may be a full desktop ID or its final dot-separated segment; matching is case-insensitive, ambiguous aliases are rejected, and an unresolved explicit `app_id` returns an empty scoped tree rather than the desktop. Without a target the result is the whole desktop AT-SPI tree, `tree_scoped` is `false`, and `message` warns; that can flood context. Use `include_screenshot: false` when the accessibility tree is sufficient. If `accessibility_tree_truncated` is `true`, the tree is incomplete: scope to a narrower target and raise `max_nodes` or `max_depth` (hard caps 10000 and 64) rather than lowering them. A tree too large to return inline comes back as its leading nodes plus `accessibility_tree_spill.path`, a private JSON file with the whole tree and the same `element_index` values: search it (for a role, name or text) instead of requesting the tree again, or page through it with `get_tree_page` when you cannot read files. Nodes are compact: null, empty, zero and false fields are left out and `actions` lists action names; `verbose: true` returns every field. To find controls in a large tree, pass `interactive_only: true`: it keeps nodes with an action, editable text or a value, plus headings, status and live regions, windows and dialogs, collapses containers, and keeps the same `element_index` values. A `title` or `window_id` target narrows a Chromium-style app with several windows under one AT-SPI root to that window, keeping the app's popups and dialogs. The compact readiness block identifies missing setup.
|
|
144
|
+
2. Begin every desktop-control turn with `get_app_state`, scoped to the app you are working in: pass `app_name_or_bundle_identifier` or a window target (`window_id`, `pid`, `app_id`, `wm_class`, `title`). An `app_id` may be a full desktop ID or its final dot-separated segment; matching is case-insensitive, ambiguous aliases are rejected, and an unresolved explicit `app_id` returns an empty scoped tree rather than the desktop. Without a target the result is the whole desktop AT-SPI tree, `tree_scoped` is `false`, and `message` warns; that can flood context. Use `include_screenshot: false` when the accessibility tree is sufficient. If `accessibility_tree_truncated` is `true`, the tree is incomplete: scope to a narrower target and raise `max_nodes` or `max_depth` (hard caps 10000 and 64) rather than lowering them. A tree too large to return inline comes back as its leading nodes plus `accessibility_tree_spill.path`, a private JSON file with the whole tree and the same `element_index` values: search it (for a role, name or text) instead of requesting the tree again, or page through it with `get_tree_page` when you cannot read files. Nodes are compact: null, empty, zero and false fields are left out and `actions` lists action names; `verbose: true` returns every field. To find controls in a large tree, pass `interactive_only: true`: it keeps nodes with an action, editable text or a value, plus headings, status and live regions, windows and dialogs, collapses containers, and keeps the same `element_index` values. A `title` or `window_id` target narrows a Chromium-style app with several windows under one AT-SPI root to that window, keeping the app's popups and dialogs. A window-targeted screenshot needs that window visible and already focused (Hyprland background mode excepted); otherwise `screenshot_error` is set and only the tree returns, so use `screenshot` to raise and capture it. The compact readiness block identifies missing setup.
|
|
143
145
|
3. Use `doctor` only when you need the full diagnostic report.
|
|
144
146
|
4. If `can_build_accessibility_tree` is false, run `setup_accessibility` and restart the target app.
|
|
145
|
-
5. If `can_query_windows` is false on GNOME Wayland, run `setup_window_targeting` and ask the user to log out and back in if setup says the shell extension needs a reload.
|
|
147
|
+
5. If `can_query_windows` or `can_focus_windows` is false on GNOME Wayland, run `setup_window_targeting` and ask the user to log out and back in if setup says the shell extension needs a reload.
|
|
146
148
|
6. Before targeted input, call `list_windows` or `focused_window` and verify the intended window by title, app id, pid, or wm class.
|
|
147
|
-
7. Prefer semantic targeting from `get_app_state`: use element indices or role/name/text/states selectors. Automatic recovery of a dead semantic target requires verified PID/window ownership and invalidates cached indices; call `get_app_state` again before further index-based actions. When several apps are in play, pass the app target (`pid`, `window_id`, `app_id`, `wm_class`, `window_title`) to `click`, `scroll`, `perform_action`, and `set_value`: an index or selector from another app's snapshot is then refused instead of acting on that app. Where the window backend reports no pid (i3 without `xprop`, COSMIC), call `get_app_state` for the same window first or pass `pid`, or the call is refused.
|
|
149
|
+
7. Prefer semantic targeting from `get_app_state`: use element indices or role/name/text/states selectors. Automatic recovery of a dead semantic target requires verified PID/window ownership and invalidates cached indices; call `get_app_state` again before further index-based actions. When several apps are in play, pass the app target (`pid`, `window_id`, `app_id`, `wm_class`, `window_title`) to `click`, `scroll`, `perform_action`, and `set_value`: an index or selector from another app's snapshot is then refused instead of acting on that app. On `click` and `scroll` the target also raises and focuses that window, and nothing is sent when focus cannot be verified; on GNOME Wayland, raising a window needs the Shell extension. The title field there is `window_title`. Where the window backend reports no pid (i3 without `xprop`, COSMIC, and wlr-foreign-toplevel on stock labwc and other wlroots compositors), call `get_app_state` for the same window first or pass `pid`, or the call is refused.
|
|
148
150
|
8. Use coordinates only when the UI surface has no useful accessibility tree.
|
|
149
151
|
9. For text input, prefer `type_text` with a target selector (`window_id`, `pid`, `app_id`, `wm_class`, `title`, `tty`, `terminal_pid`, `terminal_command`, or `terminal_cwd`) rather than relying on current focus. A `window_id` is valid only as returned by a recent `list_windows`; if a call rejects it as outside 0 to 2^53-1 or matches no window, re-run `list_windows`.
|
|
150
|
-
10. After mutating actions (`click`, `drag`, `scroll`, `press_key`, `type_text`, `perform_action`, `set_value`, `activate_window`, `move_window`, `resize_window`), either pass `screenshot_after: true | "screen" | "window"` (with optional `format`, `quality`, `scale`, `max_width`, `max_height`, `max_bytes`) and/or `include_tree: true` on the action call itself to observe the result in the same tool call, or re-check state with `get_app_state`, `focused_window`, or an app-specific readback.
|
|
152
|
+
10. After mutating actions (`click`, `drag`, `scroll`, `press_key`, `type_text`, `perform_action`, `set_value`, `activate_window`, `move_window`, `resize_window`), either pass `screenshot_after: true | "screen" | "window"` (with optional `format`, `quality`, `scale`, `max_width`, `max_height`, `max_bytes`) and/or `include_tree: true` on the action call itself to observe the result in the same tool call, or re-check state with `get_app_state`, `focused_window`, or an app-specific readback. A failed action (`ok: false`) returns no post-action screenshot or tree, so call `get_app_state`; `set_value` still observes when `verified` is `false`.
|
|
151
153
|
11. For `set_value`, `verified: true` is the only exact-content confirmation. `verified: false` requires fresh observation; password fields are deliberately unverifiable. The field is omitted for numeric values and early failures.
|
|
152
154
|
|
|
153
155
|
On Hyprland, `type_text` passes one literal wtype argument after `--` for a
|
|
@@ -183,9 +185,9 @@ only a few thousand nodes before the deadline. A snapshot stops reading at 10 se
|
|
|
183
185
|
marked truncated; if the editor is still missing, hide the embedded browser pane
|
|
184
186
|
for the snapshot.
|
|
185
187
|
|
|
186
|
-
Plain left element/index/selector `click`
|
|
187
|
-
|
|
188
|
-
|
|
188
|
+
Plain left element/index/selector `click` invokes the element's first AT-SPI
|
|
189
|
+
action instead of using toolkit bounds when that action is `click`, `press`, or
|
|
190
|
+
`toggle`, avoiding coordinate conversion. This preference does not replace a coordinate click
|
|
189
191
|
with an arbitrary action name. Explicit `x`/`y`, right clicks, and double/multiple
|
|
190
192
|
clicks retain pointer semantics. Explicit coordinates default to raw screenshot
|
|
191
193
|
pixels (`coordinate_space: "capture"`). On KDE Wayland, pass
|
|
@@ -196,11 +198,19 @@ This requires both `x`/`y` and element identity; do not
|
|
|
196
198
|
combine it with `relative: true` or use it for ordinary screenshot pixels.
|
|
197
199
|
Use `perform_action` explicitly for entry `activate` or slider `jump`; `click`
|
|
198
200
|
never substitutes those actions, including when bounds are unavailable.
|
|
199
|
-
A coordinate `click` or `
|
|
200
|
-
screenshot or between monitors, where no monitor shows that part of the
|
|
201
|
-
screenshot.
|
|
202
|
-
|
|
203
|
-
before retrying.
|
|
201
|
+
A coordinate `click`, `scroll` or `drag` result warns when the point is off
|
|
202
|
+
the screenshot or between monitors, where no monitor shows that part of the
|
|
203
|
+
screenshot. A `drag` warning names its start or end coordinate. The result
|
|
204
|
+
says whether the input was sent, and the warning says where it may have
|
|
205
|
+
landed; pick a point on a monitor and check the screen before retrying. A KWin
|
|
206
|
+
element click, including one with `coordinate_space: "atspi"`, gets no such
|
|
207
|
+
warning, since KWin maps its point onto an output. On the portal route, a
|
|
208
|
+
refused point also closes the portal session; the next pointer call starts a
|
|
209
|
+
new one, which can show the consent dialog again.
|
|
210
|
+
|
|
211
|
+
The uinput pointer is sized to the latest full screenshot. After the desktop
|
|
212
|
+
size changes, take a new screenshot before coordinate input. If replacing the
|
|
213
|
+
device fails, nothing is sent and the result says where a retry goes.
|
|
204
214
|
|
|
205
215
|
### Screenshot-relative coordinates
|
|
206
216
|
|
|
@@ -218,9 +228,9 @@ directly to a window-relative click.
|
|
|
218
228
|
|
|
219
229
|
- Already-running GTK, Qt, and Electron apps may need a restart after AT-SPI is enabled.
|
|
220
230
|
- GNOME may show a portal prompt on the first screenshot or `get_app_state` call with screenshots enabled.
|
|
221
|
-
- Inside a Selkies container, `PIXELFLUX_CU` plus `PIXELFLUX_CU_TOKEN` sends screenshots and input to pixelflux on loopback, with no portal prompt. A closed port falls through. A server error does not, so do not retry that call through another backend.
|
|
231
|
+
- Inside a Selkies container, `PIXELFLUX_CU` plus `PIXELFLUX_CU_TOKEN` sends screenshots and input to pixelflux on loopback, with no portal prompt. A closed port falls through. A server error does not, so do not retry that call through another backend. pixelflux takes left, right and middle clicks with counts 1 to 3, and multi-clicks only with the left button; other clicks go to the next backend. Its results carry no off-screen or monitor-gap warning.
|
|
222
232
|
- On GNOME Wayland, the remote-control portal asks on each new `computer-use-linux` process. `COMPUTER_USE_LINUX_PERSIST_REMOTE_DESKTOP=1` in the server environment asks the portal to remember the grant (GNOME starts with the remember box checked). Later processes reuse a single-use restore token stored mode 0600 in the user state directory. Unset is the default and prompts every time. The flag does nothing when ydotool, not the portal, is the input backend.
|
|
223
|
-
- On Hyprland, `COMPUTER_USE_LINUX_HYPRLAND_BACKGROUND=1` lets targeted `type_text`, `press_key`, `get_app_state` and `screenshot` work on a window without focusing it, including one on a hidden workspace. Pass `window_id` (or another target) on every call. An AT-SPI action can move focus inside the app, so re-focus a field with its `activate` action before typing into it. Text must be printable
|
|
233
|
+
- On Hyprland, `COMPUTER_USE_LINUX_HYPRLAND_BACKGROUND=1` lets targeted `type_text`, `press_key`, `get_app_state` and `screenshot` work on a window without focusing it, including one on a hidden workspace. Pass `window_id` (or another target) on every call. An AT-SPI action can move focus inside the app, so re-focus a field with its `activate` action before typing into it. The active Hyprland keymap must be US, or background `type_text` and `press_key` calls are refused; background `get_app_state` and `screenshot` still capture. Text must be printable ASCII, tab or newline. An unfocused Chromium ignores browser accelerators such as Ctrl+R, and Chromium joins AT-SPI only when started with `ACCESSIBILITY_ENABLED=1` if the accessibility bus reports itself disabled.
|
|
224
234
|
- Desktop input is stateful. Avoid concurrent tool calls against this MCP server.
|
|
225
235
|
- Pi serializes the native Computer Use tools and keeps one process for the session. If that process exits, do not replay an ambiguous mutating call; obtain a fresh `get_app_state` before another element-based action.
|
|
226
236
|
- `click`, `drag`, `press_key`, `type_text`, `perform_action`, and `set_value` can change real application state.
|
|
@@ -15,17 +15,20 @@ hermes mcp configure computer-use-linux
|
|
|
15
15
|
|
|
16
16
|
`configure` opens Hermes' tool-selection UI for this MCP server.
|
|
17
17
|
|
|
18
|
-
The generated config
|
|
18
|
+
The generated config holds `command`, `args` and `enabled`, plus
|
|
19
|
+
`tools.include` when you pick a subset of tools:
|
|
19
20
|
|
|
20
21
|
```yaml
|
|
21
22
|
mcp_servers:
|
|
22
23
|
computer-use-linux:
|
|
23
24
|
command: computer-use-linux
|
|
24
25
|
args: ["mcp"]
|
|
25
|
-
|
|
26
|
-
connect_timeout: 30
|
|
26
|
+
enabled: true
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
+
Hermes defaults to a 300-second tool-call timeout and a 60-second connect
|
|
30
|
+
timeout. Add `timeout` or `connect_timeout` under the server to change them.
|
|
31
|
+
|
|
29
32
|
If the binary is not on `PATH`, pass the absolute path to `--command`.
|
|
30
33
|
|
|
31
34
|
Hermes registers tools using the `mcp_<server>_<tool>` pattern. With this config, tool names are prefixed as `mcp_computer_use_linux_`, for example:
|
|
@@ -46,6 +46,9 @@ Or search by capability:
|
|
|
46
46
|
computer_use_linux_tools({ query: "inspect a window and type text" })
|
|
47
47
|
```
|
|
48
48
|
|
|
49
|
+
Called with no arguments, the loader enables `doctor`, `list_windows`,
|
|
50
|
+
`focused_window`, `get_app_state` and `get_tree_page`.
|
|
51
|
+
|
|
49
52
|
The selected native tools appear starting on the next model turn with their
|
|
50
53
|
full upstream schemas and remain active for the session:
|
|
51
54
|
|
|
@@ -87,6 +90,9 @@ for (const block of result.images) image(block);
|
|
|
87
90
|
Native Computer Use tools execute sequentially. Cancellation is forwarded to
|
|
88
91
|
the MCP request. If the server process exits, the failed call is never replayed
|
|
89
92
|
automatically; call `get_app_state` again before another element-based action.
|
|
93
|
+
Each call times out after 60 seconds. Set `COMPUTER_USE_LINUX_TIMEOUT_MS` in
|
|
94
|
+
Pi's environment to change it, for example for `run_shell` calls with
|
|
95
|
+
`timeout_seconds` above 60.
|
|
90
96
|
|
|
91
97
|
## Migration from the adapter-based package
|
|
92
98
|
|
|
@@ -110,14 +116,18 @@ legacy entry.
|
|
|
110
116
|
|
|
111
117
|
The extension looks for `computer-use-linux` in this order:
|
|
112
118
|
|
|
113
|
-
1. `COMPUTER_USE_LINUX_BIN
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
119
|
+
1. `COMPUTER_USE_LINUX_BIN`, when it is an absolute path to an executable. A
|
|
120
|
+
relative or non-executable value is ignored and the next step is used.
|
|
121
|
+
2. The binary downloaded inside the installed npm package. When it is missing,
|
|
122
|
+
as after npm 12 skipped the install script, the first real tool call runs the
|
|
123
|
+
package's `npm/install.js` to download it.
|
|
124
|
+
3. `computer-use-linux` on `PATH`, from `npm install -g` or `cargo install`,
|
|
125
|
+
when that download fails
|
|
126
|
+
|
|
127
|
+
The extension still checks that the server version and tool catalog match, so
|
|
128
|
+
a mismatched install fails with that reason instead of starting. Set
|
|
129
|
+
`COMPUTER_USE_LINUX_BIN` to run a `PATH` or development build instead of the
|
|
130
|
+
download.
|
|
121
131
|
|
|
122
132
|
Reinstall the package if the bundled binary is missing:
|
|
123
133
|
|