@xagent-ai/cli 1.0.1

package/dist/gui-subagent/action-parser/actionParser.js

@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { UITarsModelVersion, } from './types.js';
+import { MAX_RATIO, IMAGE_FACTOR, MIN_PIXELS, MAX_PIXELS_V1_5, } from './constants.js';
+import isNumber from 'lodash.isnumber';
+function roundByFactor(num, factor) {
+    return Math.round(num / factor) * factor;
+}
+function floorByFactor(num, factor) {
+    return Math.floor(num / factor) * factor;
+}
+function ceilByFactor(num, factor) {
+    return Math.ceil(num / factor) * factor;
+}
+function smartResizeForV15(height, width, maxRatio = MAX_RATIO, factor = IMAGE_FACTOR, minPixels = MIN_PIXELS, maxPixels = MAX_PIXELS_V1_5) {
+    if (Math.max(height, width) / Math.min(height, width) > maxRatio) {
+        console.error(`absolute aspect ratio must be smaller than ${maxRatio}, got ${Math.max(height, width) / Math.min(height, width)}`);
+        return null;
+    }
+    let wBar = Math.max(factor, roundByFactor(width, factor));
+    let hBar = Math.max(factor, roundByFactor(height, factor));
+    if (hBar * wBar > maxPixels) {
+        const beta = Math.sqrt((height * width) / maxPixels);
+        hBar = floorByFactor(height / beta, factor);
+        wBar = floorByFactor(width / beta, factor);
+    }
+    else if (hBar * wBar < minPixels) {
+        const beta = Math.sqrt(minPixels / (height * width));
+        hBar = ceilByFactor(height * beta, factor);
+        wBar = ceilByFactor(width * beta, factor);
+    }
+    return [wBar, hBar];
+}
+export function actionParser(params) {
+    const { prediction, factor, mode, screenContext, scaleFactor, modelVer } = params;
+    const parsed = parseActionVlm(prediction, Array.isArray(factor) ? factor : [factor, factor], mode, screenContext, scaleFactor, modelVer);
+    return {
+        parsed,
+    };
+}
+export function parseActionVlm(text, factors = [1000, 1000], mode = 'bc', screenContext, scaleFactor, modelVer = UITarsModelVersion.V1_0) {
+    let reflection = null;
+    let thought = null;
+    let actionStr = '';
+    let smartResizeFactors = null;
+    if (modelVer === UITarsModelVersion.V1_5 &&
+        screenContext?.height &&
+        screenContext?.width) {
+        smartResizeFactors = smartResizeForV15(screenContext.height, screenContext.width);
+    }
+    text = text.trim();
+    if (mode === 'bc') {
+        if (text.includes('Thought:')) {
+            const thoughtMatch = text.match(/Thought: ([\s\S]+?)(?=\s*Action[:：]|$)/);
+            if (thoughtMatch) {
+                thought = thoughtMatch[1].trim();
+            }
+        }
+        else if (text.startsWith('Reflection:')) {
+            const reflectionMatch = text.match(/Reflection: ([\s\S]+?)Action_Summary: ([\s\S]+?)(?=\s*Action[:：]|$)/);
+            if (reflectionMatch) {
+                thought = reflectionMatch[2].trim();
+                reflection = reflectionMatch[1].trim();
+            }
+        }
+        else if (text.startsWith('Action_Summary:')) {
+            const summaryMatch = text.match(/Action_Summary: (.+?)(?=\s*Action[:：]|$)/);
+            if (summaryMatch) {
+                thought = summaryMatch[1].trim();
+            }
+        }
+        if (!['Action:', 'Action：'].some((keyword) => text.includes(keyword))) {
+            actionStr = text;
+        }
+        else {
+            const actionParts = text.split(/Action[:：]/);
+            actionStr = actionParts[actionParts.length - 1];
+        }
+    }
+    else if (mode === 'o1') {
+        const thoughtMatch = text.match(/<Thought>\s*(.*?)\s*<\/Thought>/);
+        const actionSummaryMatch = text.match(/\nAction_Summary:\s*(.*?)\s*Action:/);
+        const actionMatch = text.match(/\nAction:\s*(.*?)\s*<\/Output>/);
+        const thoughtContent = thoughtMatch ? thoughtMatch[1] : null;
+        const actionSummaryContent = actionSummaryMatch
+            ? actionSummaryMatch[1]
+            : null;
+        const actionContent = actionMatch ? actionMatch[1] : null;
+        thought = `${thoughtContent}\n<Action_Summary>\n${actionSummaryContent}`;
+        actionStr = actionContent || '';
+    }
+    const allActions = actionStr.split('\n\n');
+    const actions = [];
+    for (const rawStr of allActions) {
+        const actionInstance = parseAction(rawStr.replace(/\n/g, String.raw `\n`).trimStart());
+        let actionType = '';
+        let actionInputs = {};
+        if (actionInstance) {
+            actionType = actionInstance.function;
+            const params = actionInstance.args;
+            actionInputs = {};
+            for (const [paramName, param] of Object.entries(params)) {
+                if (!param)
+                    continue;
+                const trimmedParam = param.trim();
+                if (paramName.includes('start_box') || paramName.includes('end_box')) {
+                    const oriBox = trimmedParam;
+                    const numbers = oriBox
+                        .replace(/[()[\]]/g, '')
+                        .split(',')
+                        .filter((ori) => ori !== '');
+                    const floatNumbers = numbers.map((num, idx) => {
+                        const factorIndex = idx % 2;
+                        if (modelVer === UITarsModelVersion.V1_5 && smartResizeFactors) {
+                            return Number.parseFloat(num) / smartResizeFactors[factorIndex];
+                        }
+                        return Number.parseFloat(num) / factors[factorIndex];
+                    });
+                    if (floatNumbers.length === 2) {
+                        floatNumbers.push(floatNumbers[0], floatNumbers[1]);
+                    }
+                    actionInputs[paramName.trim()] = JSON.stringify(floatNumbers);
+                    if (screenContext?.width && screenContext?.height) {
+                        const boxKey = paramName.includes('start_box')
+                            ? 'start_coords'
+                            : 'end_coords';
+                        const [x1, y1, x2 = x1, y2 = y1] = floatNumbers;
+                        const [widthFactor, heightFactor] = factors;
+                        actionInputs[boxKey] = [x1, y1, x2, y2].every(isNumber)
+                            ? [
+                                (Math.round(((x1 + x2) / 2) * screenContext?.width * widthFactor) /
+                                    widthFactor) *
+                                    (scaleFactor ?? 1),
+                                (Math.round(((y1 + y2) / 2) * screenContext?.height * heightFactor) /
+                                    heightFactor) *
+                                    (scaleFactor ?? 1),
+                            ]
+                            : [];
+                    }
+                }
+                else {
+                    actionInputs[paramName.trim()] = trimmedParam;
+                }
+            }
+        }
+        actions.push({
+            reflection: reflection,
+            thought: thought || '',
+            action_type: actionType,
+            action_inputs: actionInputs,
+        });
+    }
+    return actions;
+}
+function parseAction(actionStr) {
+    try {
+        actionStr = actionStr.replace(/<\|box_start\|>|<\|box_end\|>/g, '');
+        actionStr = actionStr
+            .replace(/(?<!start_|end_)point=/g, 'start_box=')
+            .replace(/start_point=/g, 'start_box=')
+            .replace(/end_point=/g, 'end_box=');
+        const functionPattern = /^(\w+)\((.*)\)$/;
+        const match = actionStr.trim().match(functionPattern);
+        if (!match) {
+            throw new Error('Not a function call');
+        }
+        const [_, functionName, argsStr] = match;
+        const kwargs = {};
+        if (argsStr.trim()) {
+            const argPairs = argsStr.match(/([^,']|'[^']*')+/g) || [];
+            for (const pair of argPairs) {
+                const [key, ...valueParts] = pair.split('=');
+                if (!key)
+                    continue;
+                let value = valueParts
+                    .join('=')
+                    .trim()
+                    .replace(/^['"]|['"]$/g, '');
+                if (value.includes('<bbox>')) {
+                    value = value.replace(/<bbox>|<\/bbox>/g, '').replace(/\s+/g, ',');
+                    value = `(${value})`;
+                }
+                if (value.includes('<point>')) {
+                    value = value.replace(/<point>|<\/point>/g, '').replace(/\s+/g, ',');
+                    value = `(${value})`;
+                }
+                //@ts-ignore
+                kwargs[key.trim()] = value;
+            }
+        }
+        return {
+            function: functionName,
+            args: kwargs,
+        };
+    }
+    catch (e) {
+        console.warn(`[ActionParser] Skipping invalid action: '${actionStr}'`);
+        return null;
+    }
+}
+//# sourceMappingURL=actionParser.js.map

package/dist/gui-subagent/action-parser/actionParser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actionParser.js","sourceRoot":"","sources":["../../../src/gui-subagent/action-parser/actionParser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAGL,kBAAkB,GACnB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,SAAS,EACT,YAAY,EACZ,UAAU,EACV,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,QAAQ,MAAM,iBAAiB,CAAC;AAEvC,SAAS,aAAa,CAAC,GAAW,EAAE,MAAc;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,aAAa,CAAC,GAAW,EAAE,MAAc;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,GAAW,EAAE,MAAc;IAC/C,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;AAC1C,CAAC;AAED,SAAS,iBAAiB,CACxB,MAAc,EACd,KAAa,EACb,WAAmB,SAAS,EAC5B,SAAiB,YAAY,EAC7B,YAAoB,UAAU,EAC9B,YAAoB,eAAe;IAEnC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,EAAE,CAAC;QACjE,OAAO,CAAC,KAAK,CACX,8CAA8C,QAAQ,SACpD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAClD,EAAE,CACH,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAE3D,IAAI,IAAI,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QACrD,IAAI,GAAG,aAAa,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,GAAG,aAAa,CAAC,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;SAAM,IAAI,IAAI,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;QACrD,IAAI,GAAG,YAAY,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3C,IAAI,GAAG,YAAY,CAAC,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAU5B;IAGC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,QAAQ,EAAE,GACtE,MAAM,CAAC;IAET,MAAM,MAAM,GAAG,cAAc,CAC3B,UAAU,EACV,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EACjD,IAAI,EACJ,aAAa,EACb,WAAW,EACX,QAAQ,CACT,CAAC;IAEF,OAAO;QACL,MAAM;KACP,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,UAA4B,CAAC,IAAI,EAAE,IAAI,CAAC,EACxC,OAAoB,IAAI,EACxB,aAGC,EACD,WAAoB,EACpB,WAA+B,kBAAkB,CAAC,IAAI;IAEtD,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,IAAI,SAAS,GAAG,EAAE,CAAC;IAEnB,IAAI,kBAAkB,GAA4B,IAAI,CAAC;IACvD,IACE,QAAQ,KAAK,kBAAkB,CAAC,IAAI;QACpC,aAAa,EAAE,MAAM;QACrB,aAAa,EAAE,KAAK,EACpB,CAAC;QACD,kBAAkB,GAAG,iBAAiB,CACpC,aAAa,CAAC,MAAM,EACpB,aAAa,CAAC,KAAK,CACpB,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACnB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAC7B,wCAAwC,CACzC,CAAC;YAEF,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnC,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YAC1C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAChC,qEAAqE,CACtE,CAAC;YACF,IAAI,eAAe,EAAE,CAAC;gBACpB,OAAO,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACpC,UAAU,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzC,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAC7B,0CAA0C,CAC3C,CAAC;YACF,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YACtE,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAC7C,SAAS,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;SAAM,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACnE,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CACnC,qCAAqC,CACtC,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAEjE,MAAM,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7D,MAAM,oBAAoB,GAAG,kBAAkB;YAC7C,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;YACvB,CAAC,CAAC,IAAI,CAAC;QACT,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE1D,OAAO,GAAG,GAAG,cAAc,uBAAuB,oBAAoB,EAAE,CAAC;QACzE,SAAS,GAAG,aAAa,IAAI,EAAE,CAAC;IAClC,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAA,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;QACtF,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,YAAY,GAAiB,EAAE,CAAC;QAEpC,IAAI,cAAc,EAAE,CAAC;YACnB,UAAU,GAAG,cAAc,CAAC,QAAQ,CAAC;YACrC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC;YACnC,YAAY,GAAG,EAAE,CAAC;YAElB,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxD,IAAI,CAAC,KAAK;oBAAE,SAAS;gBACrB,MAAM,YAAY,GAAI,KAAgB,CAAC,IAAI,EAAE,CAAC;gBAE9C,IAAI,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACrE,MAAM,MAAM,GAAG,YAAY,CAAC;oBAC5B,MAAM,OAAO,GAAG,MAAM;yBACnB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;yBACvB,KAAK,CAAC,GAAG,CAAC;yBACV,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;oBAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;wBAC5C,MAAM,WAAW,GAAG,GAAG,GAAG,CAAC,CAAC;wBAC5B,IAAI,QAAQ,KAAK,kBAAkB,CAAC,IAAI,IAAI,kBAAkB,EAAE,CAAC;4BAC/D,OAAO,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;wBAClE,CAAC;wBACD,OAAO,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;oBACvD,CAAC,CAAC,CAAC;oBAEH,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC9B,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;oBACtD,CAAC;oBAED,YAAY,CACV,SAAS,CAAC,IAAI,EAGb,CACF,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;oBAEjC,IAAI,aAAa,EAAE,KAAK,IAAI,aAAa,EAAE,MAAM,EAAE,CAAC;wBAClD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC;4BAC5C,CAAC,CAAC,cAAc;4BAChB,CAAC,CAAC,YAAY,CAAC;wBACjB,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,YAAY,CAAC;wBAChD,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC,GAAG,OAAO,CAAC;wBAE5C,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;4BACrD,CAAC,CAAC;gCACE,CAAC,IAAI,CAAC,KAAK,CACT,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,KAAK,GAAG,WAAW,CACrD;oCACC,WAAW,CAAC;oCACZ,CAAC,WAAW,IAAI,CAAC,CAAC;gCACpB,CAAC,IAAI,CAAC,KAAK,CACT,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,MAAM,GAAG,YAAY,CACvD;oCACC,YAAY,CAAC;oCACb,CAAC,WAAW,IAAI,CAAC,CAAC;6BACrB;4BACH,CAAC,CAAC,EAAE,CAAC;oBACT,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,YAAY,CACV,SAAS,CAAC,IAAI,EAGb,CACF,GAAG,YAAY,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,UAAU,EAAE,UAAU;YACtB,OAAO,EAAE,OAAO,IAAI,EAAE;YACtB,WAAW,EAAE,UAAU;YACvB,aAAa,EAAE,YAAY;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,SAAiB;IACpC,IAAI,CAAC;QACH,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,gCAAgC,EAAE,EAAE,CAAC,CAAC;QAEpE,SAAS,GAAG,SAAS;aAClB,OAAO,CAAC,yBAAyB,EAAE,YAAY,CAAC;aAChD,OAAO,CAAC,eAAe,EAAE,YAAY,CAAC;aACtC,OAAO,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;QAEtC,MAAM,eAAe,GAAG,iBAAiB,CAAC;QAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEtD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,GAAG,KAAK,CAAC;QAEzC,MAAM,MAAM,GAAG,EAAE,CAAC;QAElB,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;YAE1D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7C,IAAI,CAAC,GAAG;oBAAE,SAAS;gBAEnB,IAAI,KAAK,GAAG,UAAU;qBACnB,IAAI,CAAC,GAAG,CAAC;qBACT,IAAI,EAAE;qBACN,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;gBAE/B,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;oBACnE,KAAK,GAAG,IAAI,KAAK,GAAG,CAAC;gBACvB,CAAC;gBAED,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC9B,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;oBACrE,KAAK,GAAG,IAAI,KAAK,GAAG,CAAC;gBACvB,CAAC;gBAED,YAAY;gBACZ,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,IAAI,EAAE,MAAM;SACb,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,IAAI,CAAC,4CAA4C,SAAS,GAAG,CAAC,CAAC;QACvE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/gui-subagent/action-parser/constants.d.ts

@@ -0,0 +1,8 @@
+export declare const IMAGE_FACTOR = 28;
+export declare const DEFAULT_FACTOR = 1000;
+export declare const MIN_PIXELS: number;
+export declare const MAX_PIXELS_V1_0: number;
+export declare const MAX_PIXELS_DOUBAO: number;
+export declare const MAX_PIXELS_V1_5: number;
+export declare const MAX_RATIO = 200;
+//# sourceMappingURL=constants.d.ts.map

package/dist/gui-subagent/action-parser/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../src/gui-subagent/action-parser/constants.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,YAAY,KAAK,CAAC;AAC/B,eAAO,MAAM,cAAc,OAAO,CAAC;AACnC,eAAO,MAAM,UAAU,QAAoC,CAAC;AAC5D,eAAO,MAAM,eAAe,QAAqC,CAAC;AAClE,eAAO,MAAM,iBAAiB,QAAqC,CAAC;AACpE,eAAO,MAAM,eAAe,QAAsC,CAAC;AACnE,eAAO,MAAM,SAAS,MAAM,CAAC"}

package/dist/gui-subagent/action-parser/constants.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export const IMAGE_FACTOR = 28;
+export const DEFAULT_FACTOR = 1000;
+export const MIN_PIXELS = 100 * IMAGE_FACTOR * IMAGE_FACTOR;
+export const MAX_PIXELS_V1_0 = 2700 * IMAGE_FACTOR * IMAGE_FACTOR;
+export const MAX_PIXELS_DOUBAO = 5120 * IMAGE_FACTOR * IMAGE_FACTOR;
+export const MAX_PIXELS_V1_5 = 16384 * IMAGE_FACTOR * IMAGE_FACTOR;
+export const MAX_RATIO = 200;
+//# sourceMappingURL=constants.js.map

package/dist/gui-subagent/action-parser/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../src/gui-subagent/action-parser/constants.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,CAAC,MAAM,YAAY,GAAG,EAAE,CAAC;AAC/B,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC;AACnC,MAAM,CAAC,MAAM,UAAU,GAAG,GAAG,GAAG,YAAY,GAAG,YAAY,CAAC;AAC5D,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,YAAY,GAAG,YAAY,CAAC;AAClE,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,YAAY,GAAG,YAAY,CAAC;AACpE,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,GAAG,YAAY,GAAG,YAAY,CAAC;AACnE,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC"}

package/dist/gui-subagent/action-parser/index.d.ts

@@ -0,0 +1,3 @@
+export { actionParser } from './actionParser.js';
+export type { PredictionParsed, ActionInputs, UITarsModelVersion } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/gui-subagent/action-parser/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/gui-subagent/action-parser/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC"}

package/dist/gui-subagent/action-parser/index.js

@@ -0,0 +1,6 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export { actionParser } from './actionParser.js';
+//# sourceMappingURL=index.js.map

package/dist/gui-subagent/action-parser/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/gui-subagent/action-parser/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/gui-subagent/action-parser/types.d.ts

@@ -0,0 +1,24 @@
+export type Coords = [number, number] | [];
+export type ActionInputs = Partial<{
+    content: string;
+    start_box: string;
+    end_box: string;
+    key: string;
+    hotkey: string;
+    direction: string;
+    start_coords: Coords;
+    end_coords: Coords;
+}>;
+export interface PredictionParsed {
+    action_inputs: ActionInputs;
+    reflection: string | null;
+    action_type: string;
+    thought: string;
+}
+export declare enum UITarsModelVersion {
+    V1_0 = "1.0",
+    V1_5 = "1.5",
+    DOUBAO_1_5_15B = "doubao-1.5-15B",
+    DOUBAO_1_5_20B = "doubao-1.5-20B"
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/gui-subagent/action-parser/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/gui-subagent/action-parser/types.ts"],"names":[],"mappings":"AAKA,MAAM,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;AAE3C,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AAEH,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,oBAAY,kBAAkB;IAC5B,IAAI,QAAQ;IACZ,IAAI,QAAQ;IACZ,cAAc,mBAAmB;IACjC,cAAc,mBAAmB;CAClC"}

package/dist/gui-subagent/action-parser/types.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export var UITarsModelVersion;
+(function (UITarsModelVersion) {
+    UITarsModelVersion["V1_0"] = "1.0";
+    UITarsModelVersion["V1_5"] = "1.5";
+    UITarsModelVersion["DOUBAO_1_5_15B"] = "doubao-1.5-15B";
+    UITarsModelVersion["DOUBAO_1_5_20B"] = "doubao-1.5-20B";
+})(UITarsModelVersion || (UITarsModelVersion = {}));
+//# sourceMappingURL=types.js.map

package/dist/gui-subagent/action-parser/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/gui-subagent/action-parser/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAsBH,MAAM,CAAN,IAAY,kBAKX;AALD,WAAY,kBAAkB;IAC5B,kCAAY,CAAA;IACZ,kCAAY,CAAA;IACZ,uDAAiC,CAAA;IACjC,uDAAiC,CAAA;AACnC,CAAC,EALW,kBAAkB,KAAlB,kBAAkB,QAK7B"}

package/dist/gui-subagent/agent/gui-agent.d.ts

@@ -0,0 +1,126 @@
+/**
+ * GUI Agent for xagent
+ * Orchestrates desktop automation with AI-powered action execution
+ * Based on UI-TARS architecture with computer control only
+ *
+ * This implementation is aligned with packages/ui-tars/sdk/src/GUIAgent.ts
+ */
+import type { PredictionParsed } from '../types/operator.js';
+import type { Operator } from '../operator/base-operator.js';
+export declare enum GUIAgentStatus {
+    INIT = "init",
+    RUNNING = "running",
+    PAUSE = "paused",
+    END = "end",
+    ERROR = "error",
+    USER_STOPPED = "user_stopped",
+    CALL_USER = "call_user"
+}
+export interface GUIAgentConfig<T extends Operator> {
+    operator: T;
+    model?: string;
+    modelBaseUrl?: string;
+    modelApiKey?: string;
+    systemPrompt?: string;
+    loopIntervalInMs?: number;
+    maxLoopCount?: number;
+    logger?: any;
+    signal?: AbortSignal;
+    onData?: (data: GUIAgentData) => void;
+    onError?: (error: Error) => void;
+    showAIDebugInfo?: boolean;
+    retry?: {
+        screenshot?: {
+            maxRetries?: number;
+            onRetry?: (e: Error) => void;
+        };
+        model?: {
+            maxRetries?: number;
+            onRetry?: (e: Error) => void;
+        };
+        execute?: {
+            maxRetries?: number;
+            onRetry?: (e: Error) => void;
+        };
+    };
+}
+export interface GUIAgentData {
+    status: GUIAgentStatus;
+    conversations: Conversation[];
+    error?: string;
+    systemPrompt?: string;
+}
+export interface Conversation {
+    from: 'human' | 'assistant';
+    value: string;
+    screenshotBase64?: string;
+    screenshotContext?: {
+        size: {
+            width: number;
+            height: number;
+        };
+        mime?: string;
+        scaleFactor: number;
+    };
+    actionType?: string;
+    actionInputs?: Record<string, any>;
+    timing?: {
+        start: number;
+        end: number;
+        cost: number;
+    };
+    predictionParsed?: PredictionParsed[];
+}
+export declare class GUIAgent<T extends Operator> {
+    private operator;
+    private readonly model;
+    private readonly modelBaseUrl;
+    private readonly modelApiKey;
+    private readonly systemPrompt;
+    private readonly loopIntervalInMs;
+    private readonly maxLoopCount;
+    private readonly logger;
+    private readonly signal?;
+    private readonly onData?;
+    private readonly onError?;
+    private readonly showAIDebugInfo;
+    private readonly retry?;
+    private isPaused;
+    private resumePromise;
+    private resolveResume;
+    private isStopped;
+    constructor(config: GUIAgentConfig<T>);
+    /**
+     * Display conversation results with formatting similar to session.ts (simplified)
+     */
+    private displayConversationResult;
+    /**
+     * Display status message
+     */
+    private displayStatus;
+    private buildSystemPrompt;
+    initialize(): Promise<void>;
+    /**
+     * Run the GUI agent with a single instruction (UI-TARS style)
+     * All operations are determined by the GUI model
+     */
+    run(instruction: string): Promise<GUIAgentData>;
+    /**
+     * Build messages for the model API
+     */
+    private buildModelMessages;
+    /**
+     * Call the model API with debug logging
+     */
+    private callModelAPI;
+    /**
+     * Get summary from prediction text
+     */
+    private getSummary;
+    pause(): void;
+    resume(): void;
+    stop(): void;
+    cleanup(): Promise<void>;
+}
+export { GUIAgentStatus as StatusEnum };
+//# sourceMappingURL=gui-agent.d.ts.map

package/dist/gui-subagent/agent/gui-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gui-agent.d.ts","sourceRoot":"","sources":["../../../src/gui-subagent/agent/gui-agent.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAKV,gBAAgB,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AAwB7D,oBAAY,cAAc;IACxB,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,KAAK,WAAW;IAChB,GAAG,QAAQ;IACX,KAAK,UAAU;IACf,YAAY,iBAAiB;IAC7B,SAAS,cAAc;CACxB;AAED,MAAM,WAAW,cAAc,CAAC,CAAC,SAAS,QAAQ;IAChD,QAAQ,EAAE,CAAC,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,KAAK,IAAI,CAAC;IACtC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,KAAK,CAAC,EAAE;QACN,UAAU,CAAC,EAAE;YACX,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;SAC9B,CAAC;QACF,KAAK,CAAC,EAAE;YACN,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;SAC9B,CAAC;QACF,OAAO,CAAC,EAAE;YACR,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;SAC9B,CAAC;KACH,CAAC;CACH;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,cAAc,CAAC;IACvB,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,GAAG,WAAW,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE;QAClB,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACnC,MAAM,CAAC,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACvC;AAQD,qBAAa,QAAQ,CAAC,CAAC,SAAS,QAAQ;IACtC,OAAO,CAAC,QAAQ,CAAI;IACpB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAA+B;IACvD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAU;IAC1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAA6B;IAEpD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,aAAa,CAA6B;IAClD,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IAiBrC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IA8BjC;;OAEG;IACH,OAAO,CAAC,aAAa;IA2BrB,OAAO,CAAC,iBAAiB;IAiCnB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAIjC;;;OAGG;IACG,GAAG,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA2drD;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAyC1B;;OAEG;YACW,YAAY;IAgK1B;;OAEG;IACH,OAAO,CAAC,UAAU;IASlB,KAAK,IAAI,IAAI;IAOb,MAAM,IAAI,IAAI;IASd,IAAI,IAAI,IAAI;IAIN,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAa/B;AAED,OAAO,EAAE,cAAc,IAAI,UAAU,EAAE,CAAC"}