@celljs/ta-computer-use 0.0.0-stage → 4.8.0

package/src/cua-driver-daemon.ts

@@ -0,0 +1,283 @@
+import { execFile } from 'node:child_process';
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync } from 'node:fs';
+import { connect } from 'node:net';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+
+const START_TIMEOUT_MS = 10000;
+const POLL_MS = 20;
+/** Unused this long, the daemon stops (about 60 MB); the next use starts it again in a few hundred ms. */
+const IDLE_MS = 10 * 60 * 1000;
+
+/** Starts the daemon: `app` is the CuaDriver.app, `args` those of its program. */
+export type CuaDriverLauncher = (app: string, args: string[], env: Readonly<Record<string, string>>) => Promise<void>;
+
+export interface CuaDriverDaemonOptions {
+    dir?: string;
+    /** Where an app inside another app's bundle is copied to run, by version. */
+    stageDir?: string;
+    /** For the daemon's environment. */
+    env?: Readonly<Record<string, string>>;
+    launch?: CuaDriverLauncher;
+    idleMs?: number;
+}
+
+/**
+ * The daemon of the Cua Driver that comes with Ta Desktop, on macOS. There its MCP server only relays to a
+ * daemon, which macOS must start as CuaDriver.app, so that the permissions (Accessibility, Screen
+ * Recording) are the app's and not Ta's. Left alone, `cua-driver mcp` starts the CuaDriver.app in
+ * /Applications, which may be another version or missing; so Ta starts its own on a socket of its own and
+ * runs `cua-driver mcp --embedded --socket`, which only relays to it. Started on first use, it stops when
+ * unused for a while or when the process that started it exits; the relays exit with it, and pi starts
+ * them again on the next call.
+ */
+export class CuaDriverDaemon {
+    readonly socket: string;
+    readonly pidFile: string;
+    readonly dir: string;
+    /** The app that runs: `source`, or its copy outside the bundle that contains it. */
+    readonly app: string;
+    protected starting?: Promise<void>;
+    protected launched = false;
+    /** The daemon on the socket is this app's: checked once for one this process did not start. */
+    protected verified = false;
+    protected idle?: ReturnType<typeof setTimeout>;
+    protected readonly launch: CuaDriverLauncher;
+
+    constructor(
+        readonly source: string,
+        protected readonly options: CuaDriverDaemonOptions = {},
+    ) {
+        this.dir = options.dir ?? join(homedir(), 'Library', 'Caches', 'ta');
+        this.app = CuaDriverDaemon.nested(source) ? join(this.stageDir, CuaDriverDaemon.version(source), 'CuaDriver.app') : source;
+        this.socket = join(this.dir, 'cua-driver.sock');
+        this.pidFile = join(this.dir, 'cua-driver.pid');
+        this.launch = options.launch ?? openApp;
+    }
+
+    /** Running, started if it is not; one left by an earlier run is used as it is. */
+    async ensure(): Promise<void> {
+        if ((await this.alive()) && (this.verified || (await this.adopt()))) {
+            this.touch();
+            return;
+        }
+        // Unused is counted from when it answers: a start slower than the idle time is not stopped.
+        clearTimeout(this.idle);
+        this.starting ??= this.start().finally(() => {
+            this.starting = undefined;
+        });
+        await this.starting;
+        this.touch();
+    }
+
+    /** Whether it answers on its socket: a connection, which costs well under a millisecond. */
+    alive(): Promise<boolean> {
+        if (!existsSync(this.socket)) {
+            return Promise.resolve(false);
+        }
+        return new Promise(resolve => {
+            const socket = connect(this.socket);
+            socket.once('connect', () => {
+                socket.destroy();
+                resolve(true);
+            });
+            socket.on('error', () => resolve(false));
+        });
+    }
+
+    /** Stops it if this process started it. Synchronous, for the `exit` event. */
+    stop(): void {
+        clearTimeout(this.idle);
+        if (!this.launched) {
+            return;
+        }
+        this.launched = false;
+        const pid = this.pid();
+        if (pid) {
+            try {
+                process.kill(pid, 'SIGTERM');
+            } catch {
+                // Gone already.
+            }
+        }
+    }
+
+    /**
+     * Stops other daemons of the same app, such as the one macOS starts again without Ta's arguments when
+     * the user lets it quit and reopen for Screen Recording. Only processes of this very program.
+     */
+    async reap(): Promise<void> {
+        const program = this.program;
+        const own = this.pid();
+        const list = await new Promise<string>(resolve => execFile('ps', ['-axo', 'pid=,command='], (_error, stdout) => resolve(String(stdout ?? ''))));
+        for (const line of list.split('\n')) {
+            const [, pid, command] = /^\s*(\d+)\s+(.*)$/.exec(line) ?? [];
+            if (pid && Number(pid) !== own && command.startsWith(`${program} serve`) && !command.includes(this.socket)) {
+                try {
+                    process.kill(Number(pid), 'SIGTERM');
+                } catch {
+                    // Gone already.
+                }
+            }
+        }
+    }
+
+    /**
+     * Whether the running daemon, left by another Ta process or an earlier run, is this app's; another
+     * copy's (another version, whose relay would not match) is stopped, to be replaced.
+     */
+    protected async adopt(): Promise<boolean> {
+        const running = await this.identify();
+        if (!running || running.executable === this.program) {
+            this.verified = true;
+            return true;
+        }
+        try {
+            process.kill(running.pid, 'SIGTERM');
+        } catch {
+            // Gone already.
+        }
+        return false;
+    }
+
+    /** The program and process of the daemon on the socket, as it says; undefined when it does not say. */
+    protected identify(): Promise<{ executable: string; pid: number } | undefined> {
+        return new Promise(resolve =>
+            execFile(
+                this.program,
+                ['call', 'check_permissions', '{"prompt":false}', '--socket', this.socket],
+                { timeout: 5000, env: { ...process.env, ...this.options.env } },
+                (_error, stdout) => {
+                    try {
+                        const { source } = JSON.parse(String(stdout)) as { source?: { executable?: string; pid?: number } };
+                        resolve(source?.executable && source.pid ? { executable: source.executable, pid: source.pid } : undefined);
+                    } catch {
+                        resolve(undefined);
+                    }
+                },
+            ),
+        );
+    }
+
+    protected get program(): string {
+        return join(this.app, 'Contents', 'MacOS', 'cua-driver');
+    }
+
+    /** That the daemon is this app's program, not another copy LaunchServices chose; else it is stopped. */
+    protected async verify(): Promise<void> {
+        const pid = this.pid();
+        const command = pid ? await this.commandOf(pid) : undefined;
+        const program = this.program;
+        if (command === undefined || command.includes(`${program} serve`)) {
+            this.verified = true;
+            return;
+        }
+        this.stop();
+        throw new Error(
+            `macOS started another copy of CuaDriver.app (${command.split(' serve')[0]}). Run \`${program} doctor --fix\`, which makes macOS forget the other copies, then try again.`,
+        );
+    }
+
+    protected commandOf(pid: number): Promise<string | undefined> {
+        return new Promise(resolve =>
+            execFile('ps', ['-o', 'command=', '-p', String(pid)], (error, stdout) => resolve(error ? undefined : String(stdout).trim())),
+        );
+    }
+
+    protected touch(): void {
+        clearTimeout(this.idle);
+        this.idle = setTimeout(() => this.stop(), this.options.idleMs ?? IDLE_MS);
+        this.idle.unref?.();
+    }
+
+    protected pid(): number | undefined {
+        try {
+            return Number(readFileSync(this.pidFile, 'utf8').trim()) || undefined;
+        } catch {
+            return undefined;
+        }
+    }
+
+    protected get stageDir(): string {
+        return this.options.stageDir ?? join(homedir(), 'Library', 'Application Support', 'Ta', 'cua-driver');
+    }
+
+    /**
+     * macOS counts an app inside another app's bundle as part of that app: its permissions would be asked
+     * for, and kept, under Ta's name, and lost with each build of Ta. Its copy outside (a clone on APFS,
+     * which takes no space) is CuaDriver.app on its own, with the permissions the user gave Cua Driver.
+     */
+    protected async stage(): Promise<void> {
+        if (this.app === this.source || existsSync(this.app)) {
+            return;
+        }
+        const versions = dirname(dirname(this.app));
+        const temp = `${dirname(this.app)}.part`;
+        rmSync(temp, { recursive: true, force: true });
+        mkdirSync(temp, { recursive: true });
+        await run('cp', ['-cR', this.source, temp]).catch(() => run('cp', ['-R', this.source, temp]));
+        rmSync(dirname(this.app), { recursive: true, force: true });
+        renameSync(temp, dirname(this.app));
+        // The copies of earlier versions, which Ta no longer comes with.
+        for (const name of readdirSync(versions)) {
+            if (join(versions, name) !== dirname(this.app)) {
+                rmSync(join(versions, name), { recursive: true, force: true });
+            }
+        }
+    }
+
+    protected async start(): Promise<void> {
+        await this.stage();
+        mkdirSync(this.dir, { recursive: true });
+        // A socket file left by a daemon that did not stop cleanly, or by the one adopt() stopped.
+        rmSync(this.socket, { force: true });
+        this.verified = false;
+        await this.launch(this.app, ['serve', '--socket', this.socket, '--pid-file', this.pidFile], this.options.env ?? {});
+        if (!this.launched) {
+            this.launched = true;
+            process.once('exit', () => this.stop());
+        }
+        const deadline = Date.now() + START_TIMEOUT_MS;
+        while (Date.now() < deadline) {
+            if (await this.alive()) {
+                return this.verify();
+            }
+            await new Promise(resolve => setTimeout(resolve, POLL_MS));
+        }
+        throw new Error(`Cua Driver did not start in ${START_TIMEOUT_MS / 1000} s: run \`${join(this.app, 'Contents/MacOS/cua-driver')} doctor\`.`);
+    }
+}
+
+export namespace CuaDriverDaemon {
+    /** Whether an app is inside another app's bundle. */
+    export function nested(app: string): boolean {
+        return /\.app\//.test(dirname(app));
+    }
+
+    /** CFBundleVersion of an app, which names its copy. */
+    export function version(app: string): string {
+        try {
+            return /<key>CFBundleVersion<\/key>\s*<string>([^<]+)<\/string>/.exec(readFileSync(join(app, 'Contents', 'Info.plist'), 'utf8'))?.[1] ?? 'current';
+        } catch {
+            return 'current';
+        }
+    }
+}
+
+const LSREGISTER = '/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/LaunchServices.framework/Versions/A/Support/lsregister';
+
+/**
+ * Through LaunchServices, in the background and as a new instance, so that it is the app that runs and
+ * macOS applies the app's permissions. LaunchServices picks a copy by bundle id, and other apps may ship
+ * their own CuaDriver.app (another version, whose relay would not match): registering this one first makes
+ * it launch this very copy.
+ */
+async function openApp(app: string, args: string[], env: Readonly<Record<string, string>>): Promise<void> {
+    await run(LSREGISTER, ['-f', app]).catch(() => undefined);
+    const vars = Object.entries(env).flatMap(([name, value]) => ['--env', `${name}=${value}`]);
+    await run('open', ['-g', '-n', '-a', app, ...vars, '--args', ...args]);
+}
+
+function run(program: string, args: string[]): Promise<string> {
+    return new Promise((resolve, reject) => execFile(program, args, (error, stdout) => (error ? reject(error) : resolve(String(stdout)))));
+}

package/src/cua-driver.spec.ts

@@ -0,0 +1,85 @@
+import { describe, expect, it } from 'bun:test';
+import { join } from 'node:path';
+import { CuaDriver, type CuaDriverHost } from './cua-driver';
+
+function host(platform: NodeJS.Platform, files: string[], path?: string, bundled?: string): CuaDriverHost & { looked: string[] } {
+    const looked: string[] = [];
+    return {
+        platform,
+        path,
+        bundled,
+        home: '/home/u',
+        looked,
+        isFile: file => {
+            looked.push(file);
+            return files.includes(file);
+        },
+    };
+}
+
+describe('CuaDriver', () => {
+    it('finds the program on PATH first, then where its installer puts it', () => {
+        const own = (path: string) => ({ path, bundled: false });
+        expect(CuaDriver.locate(host('linux', ['/opt/bin/cua-driver', '/home/u/.local/bin/cua-driver'], '/usr/bin:/opt/bin'))).toEqual(
+            own('/opt/bin/cua-driver'),
+        );
+        expect(CuaDriver.locate(host('linux', ['/home/u/.local/bin/cua-driver'], '/usr/bin'))).toEqual(own('/home/u/.local/bin/cua-driver'));
+        expect(CuaDriver.locate(host('linux', [], '/usr/bin'))).toBeUndefined();
+    });
+
+    it('looks for the app on macOS, and for the .exe on Windows', () => {
+        const app = '/Applications/CuaDriver.app/Contents/MacOS/cua-driver';
+        expect(CuaDriver.locate(host('darwin', [app]))).toEqual({ path: app, bundled: false });
+        expect(host('linux', []).looked).toEqual([]);
+        const windows = host('win32', [], 'C:\\bin;D:\\tools');
+        CuaDriver.locate(windows);
+        expect(windows.looked).toEqual([
+            join('C:\\bin', 'cua-driver.exe'),
+            join('D:\\tools', 'cua-driver.exe'),
+            join('/home/u', '.local', 'bin', 'cua-driver.exe'),
+        ]);
+    });
+
+    it('prefers the one of Ta Desktop, a version Ta is tested with', () => {
+        const bundled = '/Ta.app/Contents/Resources/cua-driver/CuaDriver.app/Contents/MacOS/cua-driver';
+        expect(CuaDriver.locate(host('darwin', [bundled, '/home/u/.local/bin/cua-driver'], '/usr/bin', bundled))).toEqual({ path: bundled, bundled: true });
+        expect(CuaDriver.locate(host('darwin', ['/home/u/.local/bin/cua-driver'], '/usr/bin', bundled))).toEqual({
+            path: '/home/u/.local/bin/cua-driver',
+            bundled: false,
+        });
+    });
+
+    it('tells the model what each system needs', () => {
+        expect(CuaDriver.prompt('win32')).toContain('administrator');
+        expect(CuaDriver.prompt('linux')).toContain('AT-SPI');
+        expect(CuaDriver.prompt('darwin')).toContain('permissions of macOS');
+        expect(CuaDriver.prompt('freebsd')).not.toContain('administrator');
+    });
+
+    it('finds the app of a program inside one', () => {
+        expect(CuaDriver.appOf('/T/CuaDriver.app/Contents/MacOS/cua-driver')).toBe('/T/CuaDriver.app');
+        expect(CuaDriver.appOf('/T/bin/cua-driver')).toBeUndefined();
+    });
+
+    it('stops at the first file it finds', () => {
+        const found = host('linux', ['/a/cua-driver'], '/a:/b:/c');
+        CuaDriver.locate(found);
+        expect(found.looked).toEqual(['/a/cua-driver']);
+    });
+
+    it('is reached by a script that names its tools, or a search that names it', () => {
+        expect(CuaDriver.reaches(['await', 'tools', 'mcp__cua_driver__click'], {})).toBe(true);
+        expect(CuaDriver.reaches(['describeNamespace', 'mcp__cua_driver'], {})).toBe(true);
+        expect(CuaDriver.reaches(['tools', 'bash'], {})).toBe(false);
+        expect(CuaDriver.reaches('all', { code: 'searchTools("click")' })).toBe(false);
+        expect(CuaDriver.reaches('all', { code: 'describeNamespace("mcp__cua_driver")' })).toBe(true);
+        expect(CuaDriver.reaches(undefined, { code: 'mcp__cua_driver' })).toBe(false);
+    });
+
+    it('says where the program is, or how to install it', () => {
+        expect(CuaDriver.status({ path: '/x/cua-driver', bundled: false }, 'darwin')).toContain('/x/cua-driver');
+        expect(CuaDriver.status({ path: '/x/cua-driver', bundled: true }, 'linux')).toContain('Ta Desktop');
+        expect(CuaDriver.status(undefined, 'linux')).toContain(CuaDriver.INSTALL.posix);
+        expect(CuaDriver.status(undefined, 'win32')).toContain(CuaDriver.INSTALL.win32);
+    });
+});

package/src/cua-driver.ts

@@ -0,0 +1,130 @@
+import { statSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+/** Where to look for Cua Driver: the parts of the system it depends on, so that tests can give their own. */
+export interface CuaDriverHost {
+    platform: NodeJS.Platform;
+    /** The `PATH` environment variable. */
+    path?: string;
+    home: string;
+    /** {@link CuaDriver.BUNDLED_ENV}. */
+    bundled?: string;
+    isFile(path: string): boolean;
+}
+
+/** The program of Cua Driver that Ta uses. */
+export interface CuaDriverProgram {
+    path: string;
+    /** The one that comes with Ta Desktop, used when the user has none of their own. */
+    bundled: boolean;
+}
+
+/**
+ * Cua Driver (https://cua.ai/cua-driver): a program that uses the desktop apps of this computer in the
+ * background, through each system's accessibility tree, as an MCP server (`cua-driver mcp`). Ta uses the
+ * one that comes with Ta Desktop, a version Ta is tested with, or else the one the user installed.
+ */
+export namespace CuaDriver {
+    /** The name of its MCP server, so its tools are `mcp__cua_driver__<tool>`. */
+    export const SERVER = 'cua-driver';
+    /** pi's namespace of the server's tools. */
+    export const NAMESPACE = 'mcp__cua_driver';
+    /** The section of the system prompt that tells the model about it. */
+    export const SECTION = 'computer_use';
+    /** The program that comes with Ta Desktop, which sets it for the `ta serve` it starts. */
+    export const BUNDLED_ENV = 'TA_CUA_DRIVER';
+
+    /** What the system prompt says, with what only one system needs. */
+    export function prompt(platform: NodeJS.Platform): string {
+        return [
+            "Cua Driver lets you use the desktop apps of this computer in the background, without taking the user's pointer or keyboard: list apps and windows, read a window's accessibility tree, click, type, scroll and take screenshots.",
+            `Its tools are \`tools.${NAMESPACE}__*\` in codemode scripts. It starts the first time a script names \`${NAMESPACE}\`: begin with \`describeNamespace("${NAMESPACE}")\` for its tools and how to use them.`,
+            'Use it only when a task needs a desktop app, not for files or commands.',
+            ...(PLATFORM_NOTES[platform] ? [PLATFORM_NOTES[platform]] : []),
+        ].join('\n');
+    }
+
+    const PLATFORM_NOTES: Partial<Record<NodeJS.Platform, string>> = {
+        darwin: 'If a call waits for the permissions of macOS, Ta has asked the user for them; do not ask again.',
+        win32: 'Windows does not let it control the windows of apps run as administrator unless Ta runs as administrator too.',
+        linux: 'It needs the desktop session of the user (X11, or Wayland with XWayland) and its accessibility bus (AT-SPI).',
+    };
+
+    /**
+     * For each run of the one of Ta Desktop: no product telemetry and no update check, which Ta's own
+     * updates take care of. The user's own install keeps the user's settings.
+     */
+    export const BUNDLED_ENV_VARS: Readonly<Record<string, string>> = { CUA_DRIVER_RS_TELEMETRY_ENABLED: '0', CUA_DRIVER_RS_UPDATE_CHECK: '0' };
+
+    /** The description of the MCP server, which pi lists in the system prompt once it is registered. */
+    export const DESCRIPTION = 'Use the desktop apps of this computer in the background (Cua Driver)';
+
+    export const INSTALL: Readonly<Record<'posix' | 'win32', string>> = {
+        posix: '/bin/bash -c "$(curl -fsSL https://cua.ai/driver/install.sh)"',
+        win32: 'irm https://cua.ai/driver/install.ps1 | iex',
+    };
+
+    /**
+     * Its program: the one of Ta Desktop, else the user's own, on `PATH` or where its installer or its macOS
+     * app puts it. Only looks at files, it runs nothing.
+     */
+    export function locate(host: CuaDriverHost = currentHost()): CuaDriverProgram | undefined {
+        if (host.bundled && host.isFile(host.bundled)) {
+            return { path: host.bundled, bundled: true };
+        }
+        const windows = host.platform === 'win32';
+        const name = windows ? 'cua-driver.exe' : 'cua-driver';
+        const dirs = (host.path ?? '').split(windows ? ';' : ':').filter(Boolean);
+        const candidates = [...dirs.map(dir => join(dir, name)), join(host.home, '.local', 'bin', name)];
+        if (host.platform === 'darwin') {
+            const app = 'CuaDriver.app/Contents/MacOS/cua-driver';
+            candidates.push(join('/Applications', app), join(host.home, 'Applications', app));
+        }
+        const own = candidates.find(candidate => host.isFile(candidate));
+        return own ? { path: own, bundled: false } : undefined;
+    }
+
+    /** The CuaDriver.app of a program inside one. */
+    export function appOf(program: string): string | undefined {
+        const inside = '/Contents/MacOS/cua-driver';
+        const app = program.slice(0, -inside.length);
+        return program.endsWith(inside) && app.endsWith('.app') ? app : undefined;
+    }
+
+    /**
+     * Whether a call may use its tools, given what the called tool says it reaches (pi's `reachesTools`):
+     * the words of a codemode script, or `all` for one that searches tools, which then has to name it.
+     */
+    export function reaches(reach: 'all' | readonly string[] | undefined, input: unknown): boolean {
+        if (reach === undefined) {
+            return false;
+        }
+        if (reach === 'all') {
+            return JSON.stringify(input ?? null).includes(NAMESPACE);
+        }
+        return reach.some(word => word.includes(NAMESPACE));
+    }
+
+    /** What `/computer-use` says. */
+    export function status(program: CuaDriverProgram | undefined, platform: NodeJS.Platform): string {
+        if (program) {
+            const which = program.bundled ? `the one that comes with Ta Desktop (${program.path})` : program.path;
+            return `Cua Driver: ${which}. Ta starts it the first time it uses a desktop app, and checks what the system needs then.`;
+        }
+        const install = platform === 'win32' ? INSTALL.win32 : INSTALL.posix;
+        return `Ta uses desktop apps through Cua Driver, which is not installed. Install it with \`${install}\`, or use Ta Desktop, which comes with it; Ta finds it on the next message, with no restart.`;
+    }
+
+    function currentHost(): CuaDriverHost {
+        return { platform: process.platform, path: process.env.PATH, home: homedir(), bundled: process.env[BUNDLED_ENV], isFile };
+    }
+
+    function isFile(path: string): boolean {
+        try {
+            return statSync(path).isFile();
+        } catch {
+            return false;
+        }
+    }
+}