@cloudcli-ai/cloudcli 1.35.0 → 1.35.1

package/dist-server/server/modules/computer-use/computer-executor.js

@@ -1,212 +0,0 @@
-import { createRequire } from 'node:module';
-const require = createRequire(import.meta.url);
-export function getNut() {
-    try {
-        return require('@nut-tree-fork/nut-js');
-    }
-    catch {
-        return null;
-    }
-}
-export function getScreenshot() {
-    try {
-        const mod = require('screenshot-desktop');
-        return mod?.default || mod;
-    }
-    catch {
-        return null;
-    }
-}
-export function getRuntimeReadiness() {
-    const nut = getNut();
-    const screenshot = getScreenshot();
-    return {
-        nut,
-        screenshot,
-        nutInstalled: Boolean(nut),
-        screenshotInstalled: typeof screenshot === 'function',
-    };
-}
-/** Reads the pixel dimensions from a PNG/JPEG buffer header without decoding it. */
-export function readImageSize(buffer) {
-    // PNG: 8-byte signature, then IHDR chunk with width/height as big-endian uint32.
-    if (buffer.length >= 24 && buffer[0] === 0x89 && buffer[1] === 0x50) {
-        return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
-    }
-    // JPEG: scan for a Start-Of-Frame marker (0xFFC0..0xFFCF, excluding C4/C8/CC).
-    if (buffer.length >= 4 && buffer[0] === 0xff && buffer[1] === 0xd8) {
-        let offset = 2;
-        while (offset + 9 < buffer.length) {
-            if (buffer[offset] !== 0xff) {
-                offset += 1;
-                continue;
-            }
-            const marker = buffer[offset + 1];
-            if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
-                return { height: buffer.readUInt16BE(offset + 5), width: buffer.readUInt16BE(offset + 7) };
-            }
-            offset += 2 + buffer.readUInt16BE(offset + 2);
-        }
-    }
-    return null;
-}
-export async function captureScreenshot() {
-    const screenshot = getScreenshot();
-    if (typeof screenshot !== 'function') {
-        throw new Error('Computer Use runtime is not available.');
-    }
-    const buffer = await screenshot({ format: 'png' });
-    return {
-        dataUrl: `data:image/png;base64,${buffer.toString('base64')}`,
-        size: readImageSize(buffer),
-    };
-}
-/** Returns the mouse coordinate space size (logical screen pixels). */
-export async function getMouseSpaceSize() {
-    const nut = getNut();
-    if (!nut) {
-        throw new Error('Computer Use runtime is not available.');
-    }
-    const width = await nut.screen.width();
-    const height = await nut.screen.height();
-    return { width, height };
-}
-/** Maps a point from screenshot/image space to the mouse coordinate space. */
-export async function toMouseSpace(target, point) {
-    const mouseSize = await getMouseSpaceSize();
-    const image = target.displaySize || mouseSize;
-    const scaleX = image.width ? mouseSize.width / image.width : 1;
-    const scaleY = image.height ? mouseSize.height / image.height : 1;
-    return {
-        x: Math.round(point.x * scaleX),
-        y: Math.round(point.y * scaleY),
-    };
-}
-/** Maps a point from the mouse coordinate space back to screenshot/image space. */
-export function toImageSpace(target, point, mouseSize) {
-    const image = target.displaySize || mouseSize;
-    const scaleX = mouseSize.width ? image.width / mouseSize.width : 1;
-    const scaleY = mouseSize.height ? image.height / mouseSize.height : 1;
-    return {
-        x: Math.round(point.x * scaleX),
-        y: Math.round(point.y * scaleY),
-    };
-}
-function nutButton(nut, button) {
-    if (button === 'right')
-        return nut.Button.RIGHT;
-    if (button === 'middle')
-        return nut.Button.MIDDLE;
-    return nut.Button.LEFT;
-}
-/** Maps a key name (xdotool-style, as Anthropic's computer tool emits) to a nut-js Key. */
-function nutKey(nut, token) {
-    const map = {
-        return: 'Enter', enter: 'Enter', esc: 'Escape', escape: 'Escape', tab: 'Tab',
-        space: 'Space', backspace: 'Backspace', delete: 'Delete', del: 'Delete', insert: 'Insert',
-        up: 'Up', down: 'Down', left: 'Left', right: 'Right',
-        home: 'Home', end: 'End', pageup: 'PageUp', page_up: 'PageUp', pagedown: 'PageDown', page_down: 'PageDown',
-        ctrl: 'LeftControl', control: 'LeftControl', alt: 'LeftAlt', shift: 'LeftShift',
-        meta: 'LeftSuper', super: 'LeftSuper', cmd: 'LeftSuper', win: 'LeftSuper',
-        capslock: 'CapsLock',
-    };
-    const lower = token.toLowerCase();
-    if (map[lower]) {
-        return nut.Key[map[lower]];
-    }
-    if (/^f([1-9]|1[0-9]|2[0-4])$/.test(lower)) {
-        return nut.Key[`F${lower.slice(1)}`];
-    }
-    if (token.length === 1) {
-        const upper = token.toUpperCase();
-        if (nut.Key[upper] !== undefined) {
-            return nut.Key[upper];
-        }
-        if (nut.Key[`Num${token}`] !== undefined && /[0-9]/.test(token)) {
-            return nut.Key[`Num${token}`];
-        }
-    }
-    throw new Error(`Unsupported key: ${token}`);
-}
-/**
- * The cross-platform OS executor. It is intentionally free of any server,
- * database, or session dependencies so it can run both inside the local server
- * process (OSS mode) and inside the standalone desktop agent (cloud relay).
- */
-export const executor = {
-    async configure() {
-        const nut = getNut();
-        if (nut) {
-            // Make actions responsive; the agent loop already paces itself with screenshots.
-            nut.mouse.config.autoDelayMs = 2;
-            nut.keyboard.config.autoDelayMs = 2;
-        }
-        return nut;
-    },
-    async cursorPosition(target) {
-        const nut = await this.configure();
-        const mouseSize = await getMouseSpaceSize();
-        const pos = await nut.mouse.getPosition();
-        return toImageSpace(target, { x: pos.x, y: pos.y }, mouseSize);
-    },
-    async moveTo(target, point) {
-        const nut = await this.configure();
-        const dest = await toMouseSpace(target, point);
-        await nut.mouse.setPosition(new nut.Point(dest.x, dest.y));
-    },
-    async click(target, button, point, doubleClick = false) {
-        const nut = await this.configure();
-        if (point) {
-            await this.moveTo(target, point);
-        }
-        if (doubleClick) {
-            await nut.mouse.doubleClick(nutButton(nut, button));
-        }
-        else {
-            await nut.mouse.click(nutButton(nut, button));
-        }
-    },
-    async drag(target, from, to, button = 'left') {
-        const nut = await this.configure();
-        const start = await toMouseSpace(target, from);
-        const end = await toMouseSpace(target, to);
-        await nut.mouse.setPosition(new nut.Point(start.x, start.y));
-        await nut.mouse.pressButton(nutButton(nut, button));
-        await nut.mouse.setPosition(new nut.Point(end.x, end.y));
-        await nut.mouse.releaseButton(nutButton(nut, button));
-    },
-    async type(text) {
-        const nut = await this.configure();
-        await nut.keyboard.type(text);
-    },
-    async pressChord(chord) {
-        const nut = await this.configure();
-        const tokens = chord.split('+').map((token) => token.trim()).filter(Boolean);
-        if (tokens.length === 0) {
-            return;
-        }
-        const keys = tokens.map((token) => nutKey(nut, token));
-        for (const key of keys) {
-            await nut.keyboard.pressKey(key);
-        }
-        for (const key of [...keys].reverse()) {
-            await nut.keyboard.releaseKey(key);
-        }
-    },
-    async scroll(target, direction, amount, point) {
-        const nut = await this.configure();
-        if (point) {
-            await this.moveTo(target, point);
-        }
-        const steps = Math.max(1, Math.round(amount));
-        if (direction === 'up')
-            await nut.mouse.scrollUp(steps);
-        else if (direction === 'down')
-            await nut.mouse.scrollDown(steps);
-        else if (direction === 'left')
-            await nut.mouse.scrollLeft(steps);
-        else
-            await nut.mouse.scrollRight(steps);
-    },
-};
-//# sourceMappingURL=computer-executor.js.map

package/dist-server/server/modules/computer-use/computer-executor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"computer-executor.js","sourceRoot":"","sources":["../../../../server/modules/computer-use/computer-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAuB/C,MAAM,UAAU,MAAM;IACpB,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;QAC1C,OAAO,GAAG,EAAE,OAAO,IAAI,GAAG,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;IACrB,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,OAAO;QACL,GAAG;QACH,UAAU;QACV,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC;QAC1B,mBAAmB,EAAE,OAAO,UAAU,KAAK,UAAU;KACtD,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,aAAa,CAAC,MAAc;IAC1C,iFAAiF;IACjF,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACpE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,CAAC;IAC7E,CAAC;IACD,+EAA+E;IAC/E,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACnE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,OAAO,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAClC,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC9F,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7F,CAAC;YACD,MAAM,IAAI,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,IAAI,OAAO,UAAU,KAAK,UAAU,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,MAAM,GAAW,MAAM,UAAU,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3D,OAAO;QACL,OAAO,EAAE,yBAAyB,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAC7D,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;IACzC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAsB,EAAE,KAAY;IACrE,MAAM,SAAS,GAAG,MAAM,iBAAiB,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC;IAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC;QAC/B,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,YAAY,CAAC,MAAsB,EAAE,KAAY,EAAE,SAAsB;IACvF,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACtE,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC;QAC/B,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,GAAQ,EAAE,MAAmB;IAC9C,IAAI,MAAM,KAAK,OAAO;QAAE,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;IAChD,IAAI,MAAM,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;IAClD,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;AACzB,CAAC;AAED,2FAA2F;AAC3F,SAAS,MAAM,CAAC,GAAQ,EAAE,KAAa;IACrC,MAAM,GAAG,GAA2B;QAClC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK;QAC5E,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ;QACzF,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;QACpD,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU;QAC1G,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW;QAC/E,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,WAAW;QACzE,QAAQ,EAAE,UAAU;KACrB,CAAC;IACF,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QACf,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7B,CAAC;IACD,IAAI,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QAClC,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAChE,OAAO,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,KAAK,CAAC,SAAS;QACb,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;QACrB,IAAI,GAAG,EAAE,CAAC;YACR,iFAAiF;YACjF,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;YACjC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAsB;QACzC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAsB,EAAE,KAAY;QAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAsB,EAAE,MAAmB,EAAE,KAAa,EAAE,WAAW,GAAG,KAAK;QACzF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QACD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAsB,EAAE,IAAW,EAAE,EAAS,EAAE,SAAsB,MAAM;QACrF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC3C,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QACpD,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAa;QAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YACtC,MAAM,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAsB,EAAE,SAA0B,EAAE,MAAc,EAAE,KAAa;QAC5F,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9C,IAAI,SAAS,KAAK,IAAI;YAAE,MAAM,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;aACnD,IAAI,SAAS,KAAK,MAAM;YAAE,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;aAC5D,IAAI,SAAS,KAAK,MAAM;YAAE,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;;YAC5D,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF,CAAC"}

package/dist-server/server/modules/computer-use/computer-semantics.service.js

@@ -1,422 +0,0 @@
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
-import { captureScreenshot, executor, } from '../../modules/computer-use/computer-executor.js';
-import { createMacOsSemanticAdapter } from '../../modules/computer-use/semantics/adapters/macos/macos-semantic-adapter.js';
-import { createWindowsSemanticAdapter } from '../../modules/computer-use/semantics/adapters/windows/windows-semantic-adapter.js';
-import { resolveSemanticHelper } from '../../modules/computer-use/semantics/helpers/semantic-helper-resolver.js';
-import { semanticSessionStore } from '../../modules/computer-use/semantics/semantic-session-store.js';
-const execFileAsync = promisify(execFile);
-const MAX_APP_STATE_ELEMENTS = 250;
-let helperAdapter;
-function readString(value) {
-    return typeof value === 'string' ? value.trim() : '';
-}
-function requireApp(input) {
-    const app = readString(input.app);
-    if (!app) {
-        throw new Error('app is required.');
-    }
-    return app;
-}
-function readNumber(value) {
-    return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
-}
-function readButton(value) {
-    return value === 'right' || value === 'middle' ? value : 'left';
-}
-function readClickCount(value) {
-    const count = readNumber(value);
-    if (count === undefined) {
-        return 1;
-    }
-    return Math.max(1, Math.min(5, Math.trunc(count)));
-}
-function readDirection(value) {
-    return value === 'up' || value === 'left' || value === 'right' ? value : 'down';
-}
-function readSessionId(input) {
-    return readString(input.sessionId) || 'default';
-}
-function centerOf(element) {
-    const bounds = element.bounds;
-    if (!bounds) {
-        return null;
-    }
-    return {
-        x: Math.round(bounds.x + bounds.width / 2),
-        y: Math.round(bounds.y + bounds.height / 2),
-    };
-}
-function getCachedElement(sessionId, app, index, stateId) {
-    return semanticSessionStore.getElement(sessionId, app, index, stateId);
-}
-function getPoint(input, sessionId, app) {
-    const x = readNumber(input.x);
-    const y = readNumber(input.y);
-    if (x !== undefined && y !== undefined) {
-        return { x, y };
-    }
-    const elementIndex = readString(input.element_index);
-    if (!elementIndex) {
-        return undefined;
-    }
-    const element = getCachedElement(sessionId, app, elementIndex, readString(input.stateId) || undefined);
-    return element ? centerOf(element) || undefined : undefined;
-}
-function getHelperAdapter() {
-    if (helperAdapter !== undefined) {
-        return helperAdapter;
-    }
-    if (process.platform !== 'darwin' && process.platform !== 'win32') {
-        helperAdapter = null;
-        return helperAdapter;
-    }
-    const resolution = resolveSemanticHelper();
-    if (!resolution.available) {
-        helperAdapter = null;
-        return helperAdapter;
-    }
-    helperAdapter = process.platform === 'darwin'
-        ? createMacOsSemanticAdapter()
-        : createWindowsSemanticAdapter();
-    return helperAdapter;
-}
-function shouldFallbackFromHelper(error) {
-    const message = error instanceof Error ? error.message : String(error);
-    return /not implemented|unavailable|not found|does not exist|timed out|not running|exited with code|failed to start/i.test(message);
-}
-async function withHelperState(sessionId, operation) {
-    const adapter = getHelperAdapter();
-    if (!adapter) {
-        return null;
-    }
-    try {
-        return semanticSessionStore.save(sessionId, await operation(adapter));
-    }
-    catch (error) {
-        if (shouldFallbackFromHelper(error)) {
-            console.warn('[ComputerSemantics] Falling back from helper:', error instanceof Error ? error.message : String(error));
-            return null;
-        }
-        throw error;
-    }
-}
-async function run(command, args, timeout = 5000) {
-    const { stdout } = await execFileAsync(command, args, {
-        timeout,
-        windowsHide: true,
-        maxBuffer: 1024 * 1024 * 4,
-    });
-    return stdout;
-}
-async function listMacApps() {
-    const script = [
-        'tell application "System Events"',
-        'set appRows to {}',
-        'repeat with p in (application processes whose background only is false)',
-        'set end of appRows to (name of p as text)',
-        'end repeat',
-        'return appRows',
-        'end tell',
-    ].join('\n');
-    const output = await run('osascript', ['-e', script]);
-    return output.split(', ')
-        .map((name) => name.trim())
-        .filter(Boolean)
-        .map((name) => ({ name, running: true }));
-}
-async function listWindowsApps() {
-    const script = [
-        'Get-Process | Where-Object { $_.MainWindowTitle } |',
-        'Select-Object ProcessName, Id, MainWindowTitle | ConvertTo-Json -Depth 3',
-    ].join(' ');
-    const output = await run('powershell.exe', ['-NoProfile', '-Command', script]);
-    const parsed = JSON.parse(output || '[]');
-    const rows = Array.isArray(parsed) ? parsed : [parsed];
-    return rows.map((row) => ({
-        name: row.ProcessName,
-        pid: row.Id,
-        windowTitle: row.MainWindowTitle,
-        running: true,
-    }));
-}
-async function listLinuxApps() {
-    try {
-        const output = await run('wmctrl', ['-lx']);
-        return output.split(/\r?\n/)
-            .map((line) => line.trim())
-            .filter(Boolean)
-            .map((line) => {
-            const parts = line.split(/\s+/);
-            return {
-                windowId: parts[0],
-                desktop: parts[1],
-                host: parts[2],
-                className: parts[3],
-                windowTitle: parts.slice(4).join(' '),
-                running: true,
-            };
-        });
-    }
-    catch {
-        const output = await run('ps', ['-eo', 'comm=']);
-        return [...new Set(output.split(/\r?\n/).map((name) => name.trim()).filter(Boolean))]
-            .slice(0, 200)
-            .map((name) => ({ name, running: true }));
-    }
-}
-async function listApps() {
-    if (process.platform === 'darwin') {
-        return listMacApps();
-    }
-    if (process.platform === 'win32') {
-        return listWindowsApps();
-    }
-    return listLinuxApps();
-}
-async function macAccessibilityTree(app) {
-    const escapedApp = app.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
-    const script = `
-on safeText(v)
-  try
-    return v as text
-  on error
-    return ""
-  end try
-end safeText
-
-on emitElement(e, depth, maxDepth, counter)
-  if depth > maxDepth then return {}
-  set rows to {}
-  try
-    set roleText to my safeText(role of e)
-  on error
-    set roleText to "element"
-  end try
-  try
-    set titleText to my safeText(title of e)
-  on error
-    set titleText to ""
-  end try
-  try
-    set valueText to my safeText(value of e)
-  on error
-    set valueText to ""
-  end try
-  try
-    set posValue to position of e
-    set sizeValue to size of e
-    set boundsText to ((item 1 of posValue) as text) & "," & ((item 2 of posValue) as text) & "," & ((item 1 of sizeValue) as text) & "," & ((item 2 of sizeValue) as text)
-  on error
-    set boundsText to ""
-  end try
-  set end of rows to ((counter as text) & tab & roleText & tab & titleText & tab & valueText & tab & boundsText)
-  if counter > ${MAX_APP_STATE_ELEMENTS} then return rows
-  try
-    repeat with childElement in UI elements of e
-      set childRows to my emitElement(childElement, depth + 1, maxDepth, counter + (count of rows))
-      set rows to rows & childRows
-      if (count of rows) > ${MAX_APP_STATE_ELEMENTS} then return rows
-    end repeat
-  end try
-  return rows
-end emitElement
-
-tell application "System Events"
-  if not (exists process "${escapedApp}") then error "App is not running: ${escapedApp}"
-  tell process "${escapedApp}"
-    set rows to {}
-    repeat with w in windows
-      set rows to rows & my emitElement(w, 0, 4, (count of rows) + 1)
-      if (count of rows) > ${MAX_APP_STATE_ELEMENTS} then exit repeat
-    end repeat
-    return rows
-  end tell
-end tell
-`;
-    const output = await run('osascript', ['-e', script], 10000);
-    return output.split(/\r?\n|, /)
-        .map((line) => line.trim())
-        .filter(Boolean)
-        .map((line, index) => {
-        const [rawIndex, role, title, value, boundsText] = line.split('\t');
-        const boundsParts = (boundsText || '').split(',').map((part) => Number.parseFloat(part));
-        const hasBounds = boundsParts.length === 4 && boundsParts.every(Number.isFinite);
-        return {
-            index: rawIndex || String(index + 1),
-            role: role || 'element',
-            title: title || undefined,
-            value: value || undefined,
-            bounds: hasBounds
-                ? { x: boundsParts[0], y: boundsParts[1], width: boundsParts[2], height: boundsParts[3] }
-                : undefined,
-        };
-    });
-}
-async function getAccessibilityTree(app) {
-    if (process.platform === 'darwin') {
-        try {
-            return { elements: await macAccessibilityTree(app) };
-        }
-        catch (error) {
-            return { elements: [], message: error instanceof Error ? error.message : String(error) };
-        }
-    }
-    return {
-        elements: [],
-        message: 'Native accessibility tree capture is not implemented for this platform yet.',
-    };
-}
-async function getAppState(sessionId, app) {
-    if (!app) {
-        throw new Error('app is required.');
-    }
-    const helperState = await withHelperState(sessionId, (adapter) => adapter.getAppState({ sessionId, app }));
-    if (helperState) {
-        return helperState;
-    }
-    const screenshot = await captureScreenshot();
-    const tree = await getAccessibilityTree(app);
-    const state = {
-        stateId: semanticSessionStore.createStateId(),
-        app,
-        platform: process.platform,
-        screenshotDataUrl: screenshot.dataUrl,
-        displaySize: screenshot.size,
-        elements: tree.elements,
-        accessibilityTree: tree.elements,
-        message: tree.message,
-    };
-    return semanticSessionStore.save(sessionId, state);
-}
-async function targetFor(sessionId, app, stateId) {
-    const cached = semanticSessionStore.getState(sessionId, app, stateId);
-    return { displaySize: cached?.displaySize || (await captureScreenshot()).size };
-}
-export const computerSemanticsService = {
-    async callTool(name, input) {
-        const sessionId = readSessionId(input);
-        switch (name) {
-            case 'list_apps': {
-                const adapter = getHelperAdapter();
-                if (adapter) {
-                    try {
-                        return { apps: await adapter.listApps(), platform: process.platform };
-                    }
-                    catch (error) {
-                        if (!shouldFallbackFromHelper(error)) {
-                            throw error;
-                        }
-                        console.warn('[ComputerSemantics] Falling back from helper:', error instanceof Error ? error.message : String(error));
-                    }
-                }
-                return { apps: await listApps(), platform: process.platform };
-            }
-            case 'get_app_state':
-                return getAppState(sessionId, readString(input.app));
-            case 'click':
-            case 'click_element': {
-                const app = requireApp(input);
-                const helperState = await withHelperState(sessionId, (adapter) => adapter.clickElement({ ...input, sessionId, app }));
-                if (helperState) {
-                    return helperState;
-                }
-                const stateId = readString(input.stateId) || undefined;
-                const point = getPoint(input, sessionId, app);
-                if (!point) {
-                    throw new Error('click requires x/y or an element_index from computer_get_app_state.');
-                }
-                const target = await targetFor(sessionId, app, stateId);
-                const button = readButton(input.mouse_button ?? input.mouseButton);
-                const clickCount = readClickCount(input.click_count ?? input.clickCount);
-                for (let index = 0; index < clickCount; index += 1) {
-                    await executor.click(target, button, point, false);
-                }
-                return getAppState(sessionId, app);
-            }
-            case 'drag': {
-                const app = requireApp(input);
-                const helperState = await withHelperState(sessionId, (adapter) => adapter.drag({ ...input, sessionId, app }));
-                if (helperState) {
-                    return helperState;
-                }
-                const stateId = readString(input.stateId) || undefined;
-                const fromX = readNumber(input.from_x);
-                const fromY = readNumber(input.from_y);
-                const toX = readNumber(input.to_x);
-                const toY = readNumber(input.to_y);
-                if (fromX === undefined || fromY === undefined || toX === undefined || toY === undefined) {
-                    throw new Error('drag requires from_x/from_y/to_x/to_y.');
-                }
-                await executor.drag(await targetFor(sessionId, app, stateId), { x: fromX, y: fromY }, { x: toX, y: toY }, readButton(input.mouse_button ?? input.mouseButton));
-                return getAppState(sessionId, app);
-            }
-            case 'scroll':
-            case 'scroll_element': {
-                const app = requireApp(input);
-                const helperState = await withHelperState(sessionId, (adapter) => adapter.scrollElement({ ...input, sessionId, app }));
-                if (helperState) {
-                    return helperState;
-                }
-                const stateId = readString(input.stateId) || undefined;
-                const point = getPoint(input, sessionId, app);
-                if (!point) {
-                    throw new Error('scroll requires x/y or an element_index from computer_get_app_state.');
-                }
-                await executor.scroll(await targetFor(sessionId, app, stateId), readDirection(input.direction), readNumber(input.pages) ?? 1, point);
-                return getAppState(sessionId, app);
-            }
-            case 'type_text': {
-                const app = requireApp(input);
-                const helperState = await withHelperState(sessionId, (adapter) => adapter.typeText({ ...input, sessionId, app }));
-                if (helperState) {
-                    return helperState;
-                }
-                await executor.type(readString(input.text));
-                return getAppState(sessionId, app);
-            }
-            case 'press_key': {
-                const app = requireApp(input);
-                const helperState = await withHelperState(sessionId, (adapter) => adapter.pressKey({ ...input, sessionId, app }));
-                if (helperState) {
-                    return helperState;
-                }
-                await executor.pressChord(readString(input.key));
-                return getAppState(sessionId, app);
-            }
-            case 'set_value': {
-                const app = requireApp(input);
-                const helperState = await withHelperState(sessionId, (adapter) => adapter.setValue({ ...input, sessionId, app }));
-                if (helperState) {
-                    return helperState;
-                }
-                const stateId = readString(input.stateId) || undefined;
-                const point = getPoint(input, sessionId, app);
-                if (!point) {
-                    throw new Error('set_value requires x/y or an element_index from computer_get_app_state.');
-                }
-                await executor.click(await targetFor(sessionId, app, stateId), 'left', point, false);
-                await executor.pressChord(process.platform === 'darwin' ? 'cmd+a' : 'ctrl+a');
-                await executor.type(readString(input.value));
-                return getAppState(sessionId, app);
-            }
-            case 'perform_secondary_action': {
-                const app = requireApp(input);
-                const helperState = await withHelperState(sessionId, (adapter) => adapter.performSecondaryAction({ ...input, sessionId, app }));
-                if (helperState) {
-                    return helperState;
-                }
-                const stateId = readString(input.stateId) || undefined;
-                const point = getPoint(input, sessionId, app);
-                if (!point) {
-                    throw new Error('perform_secondary_action requires x/y or an element_index from computer_get_app_state.');
-                }
-                await executor.click(await targetFor(sessionId, app, stateId), 'right', point, false);
-                return getAppState(sessionId, app);
-            }
-            default:
-                throw new Error(`Unknown semantic Computer Use tool: ${name}`);
-        }
-    },
-};
-//# sourceMappingURL=computer-semantics.service.js.map