@xagent-ai/cli 1.0.1

package/dist/gui-subagent/operator/browser-operator.js

@@ -0,0 +1,306 @@
+/**
+ * Browser Operator using Puppeteer
+ * Provides browser automation capabilities for gui-subagent
+ * Based on UI-TARS architecture (@ui-tars/operators/browser-operator)
+ *
+ * This implementation is aligned with packages/ui-tars/operators/browser-operator/src/browser-operator.ts
+ */
+import puppeteer from 'puppeteer';
+import { Operator, parseBoxToScreenCoords } from './base-operator.js';
+import { sleep } from '../utils.js';
+export class BrowserOperator extends Operator {
+    browser = null;
+    page = null;
+    config;
+    logger;
+    screenCtx = null;
+    constructor(options = {}) {
+        super();
+        this.config = options.config || {
+            headless: false,
+            viewport: { width: 1280, height: 800 },
+        };
+        this.logger = options.logger || console;
+    }
+    async initialize() {
+        this.logger.info('Initializing browser...');
+        try {
+            const launchOptions = {
+                headless: this.config.headless ?? false,
+                args: [
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-gpu',
+                ],
+            };
+            if (this.config.browserPath) {
+                launchOptions.executablePath = this.config.browserPath;
+            }
+            this.browser = await puppeteer.launch(launchOptions);
+            const pages = await this.browser.pages();
+            this.page = pages[0] || await this.browser.newPage();
+            await this.page.setViewport({
+                width: this.config.viewport?.width || 1280,
+                height: this.config.viewport?.height || 800,
+                deviceScaleFactor: this.config.deviceScaleFactor || 1,
+            });
+            this.screenCtx = {
+                width: this.config.viewport?.width || 1280,
+                height: this.config.viewport?.height || 800,
+                scaleFactor: this.config.deviceScaleFactor || 1,
+            };
+            this.logger.info('Browser initialized successfully');
+        }
+        catch (error) {
+            this.logger.error('Failed to initialize browser:', error);
+            throw error;
+        }
+    }
+    getSupportedActions() {
+        return [
+            'click',
+            'left_click',
+            'left_single',
+            'left_double',
+            'double_click',
+            'right_click',
+            'right_single',
+            'middle_click',
+            'mouse_move',
+            'hover',
+            'drag',
+            'scroll',
+            'type',
+            'hotkey',
+            'navigate',
+            'navigate_back',
+            'wait',
+            'finished',
+            'user_stop',
+            'error_env',
+        ];
+    }
+    screenContext() {
+        // Return default context if not initialized yet
+        // This allows the first action (e.g., navigate) to trigger lazy initialization
+        if (!this.screenCtx) {
+            return {
+                width: 1280,
+                height: 800,
+                scaleFactor: 1,
+            };
+        }
+        return this.screenCtx;
+    }
+    async screenshot() {
+        if (!this.page) {
+            throw new Error('Page not initialized');
+        }
+        try {
+            const url = this.page.url();
+            const base64 = await this.page.screenshot({
+                encoding: 'base64',
+                type: 'png',
+                fullPage: false,
+            });
+            return {
+                status: 'success',
+                base64: base64,
+                url,
+                scaleFactor: this.screenCtx?.scaleFactor || 1,
+            };
+        }
+        catch (error) {
+            this.logger.error('Screenshot failed:', error);
+            return {
+                status: 'failed',
+                errorMessage: error.message,
+            };
+        }
+    }
+    async execute(params) {
+        const { parsedPrediction, screenWidth, screenHeight, scaleFactor } = params;
+        const { action_type, action_inputs } = parsedPrediction;
+        const startBoxStr = action_inputs?.start_box || '';
+        const { x: startX, y: startY } = parseBoxToScreenCoords({
+            boxStr: startBoxStr,
+            screenWidth,
+            screenHeight,
+        });
+        try {
+            await this.executeAction(action_type, action_inputs, { startX, startY, screenWidth, screenHeight, scaleFactor });
+            return { status: 'success' };
+        }
+        catch (error) {
+            this.logger.error(`Failed to execute action ${action_type}:`, error);
+            return {
+                status: 'failed',
+                errorMessage: error.message,
+            };
+        }
+    }
+    async executeAction(actionType, inputs, context) {
+        const { startX, startY, screenWidth, screenHeight, scaleFactor } = context;
+        const page = await this.getActivePage();
+        switch (actionType) {
+            case 'wait':
+                await sleep(5000);
+                break;
+            case 'mouse_move':
+            case 'hover':
+                await page.mouse.move(startX, startY);
+                break;
+            case 'click':
+            case 'left_click':
+            case 'left_single':
+                await page.mouse.move(startX, startY);
+                await sleep(100);
+                await page.mouse.click(startX, startY);
+                break;
+            case 'left_double':
+            case 'double_click':
+                await page.mouse.move(startX, startY);
+                await sleep(100);
+                await page.mouse.click(startX, startY, { clickCount: 2 });
+                break;
+            case 'right_click':
+            case 'right_single':
+                await page.mouse.move(startX, startY);
+                await sleep(100);
+                await page.mouse.click(startX, startY, { button: 'right' });
+                break;
+            case 'middle_click':
+                await page.mouse.move(startX, startY);
+                await page.mouse.click(startX, startY, { button: 'middle' });
+                break;
+            case 'type': {
+                const content = inputs.content?.trim();
+                if (content) {
+                    const stripContent = content.replace(/\\n$/, '').replace(/\n$/, '');
+                    await page.keyboard.type(stripContent, { delay: 50 });
+                    if (content.endsWith('\n') || content.endsWith('\\n')) {
+                        await page.keyboard.press('Enter');
+                    }
+                }
+                break;
+            }
+            case 'hotkey': {
+                const keyStr = inputs?.key || inputs?.hotkey;
+                if (keyStr) {
+                    const keys = this.parseHotkeys(keyStr);
+                    for (const key of keys) {
+                        await page.keyboard.down(key);
+                    }
+                    for (let i = keys.length - 1; i >= 0; i--) {
+                        await page.keyboard.up(keys[i]);
+                    }
+                }
+                break;
+            }
+            case 'scroll': {
+                const { direction } = inputs;
+                if (startX !== null && startY !== null) {
+                    await page.mouse.move(startX, startY);
+                }
+                switch (direction?.toLowerCase()) {
+                    case 'up':
+                        await page.mouse.wheel({ deltaY: -500 });
+                        break;
+                    case 'down':
+                        await page.mouse.wheel({ deltaY: 500 });
+                        break;
+                    default:
+                        this.logger.warn(`Unsupported scroll direction: ${direction}`);
+                }
+                break;
+            }
+            case 'navigate': {
+                // Lazy initialization: initialize browser only when navigating
+                if (!this.browser) {
+                    await this.initialize();
+                }
+                let url = inputs?.url;
+                if (!url) {
+                    throw new Error('No URL specified for navigation');
+                }
+                if (!/^https?:\/\//i.test(url)) {
+                    url = 'https://' + url;
+                }
+                await this.page.goto(url, { waitUntil: 'networkidle2' });
+                break;
+            }
+            case 'navigate_back':
+                await page.goBack();
+                break;
+            case 'error_env':
+            case 'finished':
+            case 'user_stop':
+                break;
+            default:
+                this.logger.warn(`Unsupported action: ${actionType}`);
+        }
+    }
+    parseHotkeys(keyStr) {
+        const keyMap = {
+            return: 'Enter',
+            ctrl: process.platform === 'darwin' ? 'Meta' : 'Control',
+            shift: 'Shift',
+            alt: 'Alt',
+            meta: 'Meta',
+            win: 'Meta',
+            cmd: 'Meta',
+            ',': ',',
+            arrowup: 'ArrowUp',
+            arrowdown: 'ArrowDown',
+            arrowleft: 'ArrowLeft',
+            arrowright: 'ArrowRight',
+        };
+        return keyStr
+            .split(/[\s+]+/)
+            .map((k) => k.toLowerCase())
+            .map((k) => keyMap[k] || k);
+    }
+    async getActivePage() {
+        if (!this.page) {
+            throw new Error('Page not initialized');
+        }
+        return this.page;
+    }
+    async cleanup() {
+        // Don't close browser/page - let user continue using it
+        this.logger.info('GUI Agent cleanup - browser kept open for user');
+    }
+    async destroyInstance() {
+        this.logger.info('Destroying instance...');
+        await this.cleanup();
+        if (this.browser) {
+            await this.browser.close();
+            this.browser = null;
+        }
+    }
+    getBrowser() {
+        return this.browser;
+    }
+    getPage() {
+        return this.page;
+    }
+    static get MANUAL() {
+        return {
+            ACTION_SPACES: [
+                `click(point='<point>x1 y1</point>')`,
+                `left_double(point='<point>x1 y1</point>')`,
+                `right_single(point='<point>x1 y1</point>')`,
+                `drag(start_point='<point>x1 y1</point>', end_point='<point>x2 y2</point>')`,
+                `hotkey(key='ctrl c') # Split keys with a space and use lowercase.`,
+                `type(content='xxx') # Use escape characters \\', \\", and \\n. Use \\n at the end to submit.`,
+                `scroll(point='<point>x1 y1</point>', direction='down or up or right or left')`,
+                `wait() #Sleep for 5s and take a screenshot.`,
+                `finished(content='xxx') # Use escape characters \\', \\", and \\n.`,
+                `navigate(url='https://example.com')`,
+                `navigate_back()`,
+            ],
+        };
+    }
+}
+//# sourceMappingURL=browser-operator.js.map

package/dist/gui-subagent/operator/browser-operator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-operator.js","sourceRoot":"","sources":["../../../src/gui-subagent/operator/browser-operator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,SAA0D,MAAM,WAAW,CAAC;AAQnF,OAAO,EAAE,QAAQ,EAAuB,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAOpC,MAAM,OAAO,eAAgB,SAAQ,QAAQ;IACnC,OAAO,GAAmB,IAAI,CAAC;IAC/B,IAAI,GAAgB,IAAI,CAAC;IACzB,MAAM,CAAiB;IACvB,MAAM,CAAU;IAChB,SAAS,GAAyB,IAAI,CAAC;IAE/C,YAAY,UAAkC,EAAE;QAC9C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI;YAC9B,QAAQ,EAAE,KAAK;YACf,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;SACvC,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC;IAC1C,CAAC;IAES,KAAK,CAAC,UAAU;QACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAE5C,IAAI,CAAC;YACH,MAAM,aAAa,GAAkB;gBACnC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,KAAK;gBACvC,IAAI,EAAE;oBACJ,cAAc;oBACd,0BAA0B;oBAC1B,yBAAyB;oBACzB,eAAe;iBAChB;aACF,CAAC;YAEF,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC5B,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAErD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAErD,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;gBAC1B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,IAAI;gBAC1C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG;gBAC3C,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC;aACtD,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,GAAG;gBACf,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,IAAI;gBAC1C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG;gBAC3C,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC;aAChD,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,mBAAmB;QACjB,OAAO;YACL,OAAO;YACP,YAAY;YACZ,aAAa;YACb,aAAa;YACb,cAAc;YACd,aAAa;YACb,cAAc;YACd,cAAc;YACd,YAAY;YACZ,OAAO;YACP,MAAM;YACN,QAAQ;YACR,MAAM;YACN,QAAQ;YACR,UAAU;YACV,eAAe;YACf,MAAM;YACN,UAAU;YACV,WAAW;YACX,WAAW;SACZ,CAAC;IACJ,CAAC;IAES,aAAa;QACrB,gDAAgD;QAChD,+EAA+E;QAC/E,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,GAAG;gBACX,WAAW,EAAE,CAAC;aACf,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,UAAU;QACxB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;gBACxC,QAAQ,EAAE,QAAQ;gBAClB,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YAEH,OAAO;gBACL,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,MAAgB;gBACxB,GAAG;gBACH,WAAW,EAAE,IAAI,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC;aAC9C,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;YAC/C,OAAO;gBACL,MAAM,EAAE,QAAQ;gBAChB,YAAY,EAAG,KAAe,CAAC,OAAO;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,MAAqB;QAC3C,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;QAC5E,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAAC;QAExD,MAAM,WAAW,GAAG,aAAa,EAAE,SAAS,IAAI,EAAE,CAAC;QACnD,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,sBAAsB,CAAC;YACtD,MAAM,EAAE,WAAW;YACnB,WAAW;YACX,YAAY;SACb,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,aAAa,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC;YACjH,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,WAAW,GAAG,EAAE,KAAK,CAAC,CAAC;YACrE,OAAO;gBACL,MAAM,EAAE,QAAQ;gBAChB,YAAY,EAAG,KAAe,CAAC,OAAO;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,UAAkB,EAClB,MAA2B,EAC3B,OAA2G;QAE3G,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAC3E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAExC,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,MAAM;gBACT,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClB,MAAM;YAER,KAAK,YAAY,CAAC;YAClB,KAAK,OAAO;gBACV,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACtC,MAAM;YAER,KAAK,OAAO,CAAC;YACb,KAAK,YAAY,CAAC;YAClB,KAAK,aAAa;gBAChB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACtC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;gBACjB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACvC,MAAM;YAER,KAAK,aAAa,CAAC;YACnB,KAAK,cAAc;gBACjB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACtC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;gBACjB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC1D,MAAM;YAER,KAAK,aAAa,CAAC;YACnB,KAAK,cAAc;gBACjB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACtC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;gBACjB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC5D,MAAM;YAER,KAAK,cAAc;gBACjB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACtC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC7D,MAAM;YAER,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;gBACvC,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBACpE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;oBAEtD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;wBACtD,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrC,CAAC;gBACH,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,MAAM,EAAE,MAAM,CAAC;gBAC7C,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;oBACvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAU,CAAC,CAAC;oBACvC,CAAC;oBACD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC1C,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAQ,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;gBAC7B,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;oBACvC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACxC,CAAC;gBAED,QAAQ,SAAS,EAAE,WAAW,EAAE,EAAE,CAAC;oBACjC,KAAK,IAAI;wBACP,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;wBACzC,MAAM;oBACR,KAAK,MAAM;wBACT,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;wBACxC,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,SAAS,EAAE,CAAC,CAAC;gBACnE,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,+DAA+D;gBAC/D,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC1B,CAAC;gBACD,IAAI,GAAG,GAAG,MAAM,EAAE,GAAG,CAAC;gBACtB,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACrD,CAAC;gBACD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/B,GAAG,GAAG,UAAU,GAAG,GAAG,CAAC;gBACzB,CAAC;gBACD,MAAM,IAAI,CAAC,IAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC1D,MAAM;YACR,CAAC;YAED,KAAK,eAAe;gBAClB,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM;YAER,KAAK,WAAW,CAAC;YACjB,KAAK,UAAU,CAAC;YAChB,KAAK,WAAW;gBACd,MAAM;YAER;gBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,MAAc;QACjC,MAAM,MAAM,GAA2B;YACrC,MAAM,EAAE,OAAO;YACf,IAAI,EAAE,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YACxD,KAAK,EAAE,OAAO;YACd,GAAG,EAAE,KAAK;YACV,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,MAAM;YACX,GAAG,EAAE,MAAM;YACX,GAAG,EAAE,GAAG;YACR,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,WAAW;YACtB,SAAS,EAAE,WAAW;YACtB,UAAU,EAAE,YAAY;SACzB,CAAC;QAEF,OAAO,MAAM;aACV,KAAK,CAAC,QAAQ,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,wDAAwD;QACxD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,KAAc,MAAM;QACxB,OAAO;YACL,aAAa,EAAE;gBACb,qCAAqC;gBACrC,2CAA2C;gBAC3C,4CAA4C;gBAC5C,4EAA4E;gBAC5E,mEAAmE;gBACnE,8FAA8F;gBAC9F,+EAA+E;gBAC/E,6CAA6C;gBAC7C,oEAAoE;gBACpE,qCAAqC;gBACrC,iBAAiB;aAClB;SACF,CAAC;IACJ,CAAC;CACF"}

package/dist/gui-subagent/operator/computer-operator.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Computer Operator using @computer-use/nut-js
+ * Provides desktop automation capabilities for gui-subagent
+ * Based on UI-TARS NutJSOperator implementation
+ *
+ * This implementation is aligned with packages/ui-tars/operators/nut-js/src/index.ts
+ */
+import type { OperatorConfig, ScreenContext, ScreenshotOutput, ExecuteParams, ExecuteOutput } from '../types/operator.js';
+import { Operator, type OperatorManual } from './base-operator.js';
+export interface ComputerOperatorOptions {
+    config?: OperatorConfig;
+    computerConfig?: Record<string, any>;
+    logger?: any;
+}
+export declare class ComputerOperator extends Operator {
+    private config;
+    private logger;
+    private screenCtx;
+    constructor(options?: ComputerOperatorOptions);
+    protected initialize(): Promise<void>;
+    private getScreenSize;
+    getSupportedActions(): string[];
+    protected screenContext(): ScreenContext;
+    protected screenshot(): Promise<ScreenshotOutput>;
+    protected execute(params: ExecuteParams): Promise<ExecuteOutput>;
+    private executeAction;
+    cleanup(): Promise<void>;
+    destroyInstance(): Promise<void>;
+    static get MANUAL(): OperatorManual;
+}
+//# sourceMappingURL=computer-operator.d.ts.map

package/dist/gui-subagent/operator/computer-operator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"computer-operator.d.ts","sourceRoot":"","sources":["../../../src/gui-subagent/operator/computer-operator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAeH,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1H,OAAO,EAAE,QAAQ,EAAE,KAAK,cAAc,EAA0B,MAAM,oBAAoB,CAAC;AAK3F,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,GAAG,CAAC;CACd;AAED,qBAAa,gBAAiB,SAAQ,QAAQ;IAC5C,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,SAAS,CAA8B;gBAEnC,OAAO,GAAE,uBAA4B;cAMjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAkB7B,aAAa;IAiB3B,mBAAmB,IAAI,MAAM,EAAE;IA6B/B,SAAS,CAAC,aAAa,IAAI,aAAa;cAOxB,UAAU,IAAI,OAAO,CAAC,gBAAgB,CAAC;cAuCvC,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;YAwCxD,aAAa;IAgRrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAKtC,WAAoB,MAAM,IAAI,cAAc,CA2C3C;CACF"}