@beeos-ai/device-agent 0.15.0 → 0.16.0

package/dist/agent-runtime/models/vlm/guiowl.js

@@ -1,249 +0,0 @@
-import { invalidModelOutput, ModelAdapter, registerAdapter } from "./base.js";
-import { parseToolCall } from "./shared/tool-call.js";
-// ─────────────────────────────────────────────────────────────────────────────
-// GUI-Owl / GUI-Plus adapter
-//
-// Shared format used by:
-//   - mPLUG/GUI-Owl-7B/32B/1.5   (open-source, https://github.com/X-PLUG/MobileAgent)
-//   - Alibaba Tongyi gui-plus-*  (DashScope, e.g. gui-plus-2026-02-26)
-//
-// Protocol (mirrors ClawGUI prompts_guiowl.py exactly):
-//   - system prompt contains a full <tools></tools> XML schema for `mobile_use`
-//   - model output:  "Action: <short desc>\n<tool_call>{"name":"mobile_use","arguments":{...}}</tool_call>"
-//   - coordinates are 0–999 (resolution declared in the schema as 1000x1000)
-//   - only ONE user message is ever sent per turn; action history is injected as
-//     "Previous actions: Step1: ... Step2: ..." text, no assistant messages accumulated
-// ─────────────────────────────────────────────────────────────────────────────
-const SCREEN_W = 1000;
-const SCREEN_H = 1000;
-function buildToolSchema(w, h) {
-    const schema = {
-        type: "function",
-        function: {
-            name_for_human: "mobile_use",
-            name: "mobile_use",
-            description: "Use a touchscreen to interact with a mobile device, and take screenshots.\n" +
-                "* This is an interface to a mobile device with touchscreen. " +
-                "You can perform actions like clicking, typing, swiping, etc.\n" +
-                "* Some applications may take time to start or process actions, " +
-                "so you may need to wait and take successive screenshots to see the results of your actions.\n" +
-                `* The screen's resolution is ${w}x${h}.\n` +
-                "* Make sure to click any buttons, links, icons, etc with the cursor tip " +
-                "in the center of the element. Don't click boxes on their edges unless asked.",
-            parameters: {
-                properties: {
-                    action: {
-                        description: "The action to perform. The available actions are:\n" +
-                            "* `key`: Perform a key event on the mobile device.\n" +
-                            "    - This supports adb's `keyevent` syntax.\n" +
-                            '    - Examples: "volume_up", "volume_down", "power", "camera", "clear".\n' +
-                            "* `click`: Click the point on the screen with coordinate (x, y).\n" +
-                            "* `long_press`: Press the point on the screen with coordinate (x, y) for specified seconds.\n" +
-                            "* `swipe`: Swipe from the starting point with coordinate (x, y) to the end point with coordinates2 (x2, y2).\n" +
-                            "* `type`: Input the specified text into the activated input box.\n" +
-                            "* `system_button`: Press the system button.\n" +
-                            "* `open`: Open an app on the device.\n" +
-                            "* `wait`: Wait specified seconds for the change to happen.\n" +
-                            "* `answer`: Terminate the current task and output the answer.\n" +
-                            "* `interact`: Resolve the blocking window by interacting with the user.\n" +
-                            "* `terminate`: Terminate the current task and report its completion status.",
-                        enum: [
-                            "key", "click", "long_press", "swipe", "type",
-                            "system_button", "open", "wait", "answer", "interact", "terminate",
-                        ],
-                        type: "string",
-                    },
-                    coordinate: {
-                        description: "(x, y): The x (pixels from the left edge) and y (pixels from the top edge). " +
-                            "Required only by `action=click`, `action=long_press`, and `action=swipe`.",
-                        type: "array",
-                    },
-                    coordinate2: {
-                        description: "(x, y): swipe endpoint. Required only by `action=swipe`.",
-                        type: "array",
-                    },
-                    text: {
-                        description: "Required only by `action=key`, `action=type`, `action=open`, `action=answer`, and `action=interact`.",
-                        type: "string",
-                    },
-                    time: {
-                        description: "Seconds. Required only by `action=long_press` and `action=wait`.",
-                        type: "number",
-                    },
-                    button: {
-                        description: "Back = return to previous screen; Home = desktop; Menu = app background menu; Enter = press enter. " +
-                            "Required only by `action=system_button`.",
-                        enum: ["Back", "Home", "Menu", "Enter"],
-                        type: "string",
-                    },
-                    status: {
-                        description: "Task status. Required only by `action=terminate`.",
-                        type: "string",
-                        enum: ["success", "failure"],
-                    },
-                },
-                required: ["action"],
-                type: "object",
-            },
-            args_format: "Format the arguments as a JSON object.",
-        },
-    };
-    return JSON.stringify(schema);
-}
-function systemPromptEn(w, h) {
-    return `# Tools
-
-You may call one or more functions to assist with the user query.
-
-You are provided with function signatures within <tools></tools> XML tags:
-<tools>
-${buildToolSchema(w, h)}
-</tools>
-
-For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
-<tool_call>
-{"name": <function-name>, "arguments": <args-json-object>}
-</tool_call>
-
-# Response format
-
-Response format for every step:
-1) Action: a short imperative describing what to do in the UI.
-2) A single <tool_call>...</tool_call> block containing only the JSON: {"name": <function-name>, "arguments": <args-json-object>}.
-
-Rules:
-- Output exactly in the order: Action, <tool_call>.
-- Be brief: one line for Action.
-- Do not output anything else outside those two parts.
-- If finishing, use mobile_use with action=terminate in the tool call.`;
-}
-function systemPromptCn(w, h) {
-    return `# 工具
-
-你可以调用一个或多个函数来完成用户的请求。
-
-可用的函数定义在 <tools></tools> XML 标签内：
-<tools>
-${buildToolSchema(w, h)}
-</tools>
-
-每次调用函数时，请在 <tool_call></tool_call> XML 标签内返回一个包含函数名和参数的 JSON 对象：
-<tool_call>
-{"name": <函数名>, "arguments": <参数JSON对象>}
-</tool_call>
-
-# 回复格式
-
-每一步的回复格式：
-1) Action: 一句简短的祈使句，描述在UI上要执行的操作。
-2) 一个 <tool_call>...</tool_call> 代码块，只包含 JSON：{"name": <函数名>, "arguments": <参数JSON对象>}。
-
-规则：
-- 严格按照 Action、<tool_call> 的顺序输出。
-- 保持简洁：Action 只需一句话。
-- 不要输出这两部分以外的任何内容。
-- 如果任务完成，在 tool_call 中使用 mobile_use 的 action=terminate。`;
-}
-function buildUserQuery(task, history, lang, opts) {
-    const isEn = lang !== "cn";
-    const ctxHint = !opts.hasScreenshot && opts.hasXml
-        ? (isEn
-            ? "according to the UI hierarchy (XML) below, the instruction, and previous actions"
-            : "根据下方 UI 结构(XML)、指令和之前的操作")
-        : opts.hasScreenshot && opts.hasXml
-            ? (isEn
-                ? "according to the UI screenshot, the UI hierarchy (XML) below, the instruction, and previous actions"
-                : "根据 UI 截图、下方 UI 结构(XML)、指令和之前的操作")
-            : (isEn
-                ? "according to the UI screenshot, the instruction, and previous actions"
-                : "根据UI截图、指令和之前的操作");
-    if (history.length === 0) {
-        return isEn
-            ? `Please generate the next move ${ctxHint}.\n\nInstruction: ${task}`
-            : `请${ctxHint}，生成下一步操作。\n\n指令: ${task}`;
-    }
-    const prev = history.map((d, i) => `Step${i + 1}: ${d} Tool response: None`).join("\n");
-    return isEn
-        ? `Please generate the next move ${ctxHint}.\n\nInstruction: ${task}\n\nPrevious actions: \n${prev}`
-        : `请${ctxHint}，生成下一步操作。\n\n指令: ${task}\n\n之前的操作: \n${prev}`;
-}
-class GUIOwlAdapter extends ModelAdapter {
-    name = "guiowl";
-    observationProfile = {
-        defaultMode: "vision",
-        supportedModes: ["vision", "xml", "hybrid"],
-    };
-    // Matches GUI-Owl-{7B,32B,1.5} open-source AND Alibaba DashScope gui-plus-*
-    patterns = [/guiowl/, /gui[-_]?owl/, /owl.*gui/, /gui[-_]?plus/];
-    actionHistory = [];
-    buildMessages(task, screenInfo, _context, isFirst, lang) {
-        if (isFirst)
-            this.actionHistory = [];
-        const sys = lang === "cn" ? systemPromptCn(SCREEN_W, SCREEN_H) : systemPromptEn(SCREEN_W, SCREEN_H);
-        const hasShot = !!screenInfo.imageRef;
-        const hasXml = !!screenInfo.uiXml;
-        let userText = buildUserQuery(task, this.actionHistory, lang, { hasScreenshot: hasShot, hasXml });
-        if (hasXml && screenInfo.uiXml) {
-            userText += lang !== "cn"
-                ? `\n\n** UI Hierarchy (XML) **\n${screenInfo.uiXml}`
-                : `\n\n** UI 结构 (XML) **\n${screenInfo.uiXml}`;
-        }
-        const userContent = [
-            { type: "text", text: userText },
-        ];
-        if (screenInfo.imageRef) {
-            userContent.push({
-                type: "image_url",
-                image_url: { url: screenInfo.imageRef.url },
-            });
-        }
-        return [
-            { role: "system", content: sys },
-            { role: "user", content: userContent },
-        ];
-    }
-    parseResponse(raw) {
-        // Extract the "Action: ..." line as thinking/description for history
-        const actionLineMatch = raw.match(/^\s*Action:\s*(.+?)$/m);
-        const description = actionLineMatch ? actionLineMatch[1].trim().replace(/^"|"$/g, "") : "";
-        const parsed = parseToolCall(raw, description)
-            ?? invalidModelOutput(this.name, "expected a valid <tool_call> JSON action");
-        // Coordinates come back 0-999; handler treats "0-1000" the same bucket.
-        parsed.coordSystem = "0-1000";
-        return parsed;
-    }
-    updateContext(_context, _response, parsed) {
-        if (!parsed.shouldFinish) {
-            const desc = (parsed.thinking || actionSummary(parsed)).slice(0, 200);
-            if (desc)
-                this.actionHistory.push(desc);
-        }
-        // GUI-Owl / GUI-Plus rebuild messages from scratch each turn (no accumulation)
-        return [];
-    }
-    clearHistory() {
-        this.actionHistory = [];
-    }
-}
-registerAdapter(() => new GUIOwlAdapter());
-function actionSummary(parsed) {
-    const { actionType: t, params: p } = parsed;
-    if (t === "tap")
-        return `click(${p.x},${p.y})`;
-    if (t === "double_tap")
-        return `double_tap(${p.x},${p.y})`;
-    if (t === "long_press")
-        return `long_press(${p.x},${p.y})`;
-    if (t === "type")
-        return `type("${String(p.text).slice(0, 40)}")`;
-    if (t === "swipe")
-        return `swipe(${p.start_x},${p.start_y}→${p.end_x},${p.end_y})`;
-    if (t === "open_app")
-        return `open(${p.app_name})`;
-    if (t === "press_back")
-        return "system_button(Back)";
-    if (t === "press_home")
-        return "system_button(Home)";
-    return t;
-}
-//# sourceMappingURL=guiowl.js.map

package/dist/agent-runtime/models/vlm/guiowl.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"guiowl.js","sourceRoot":"","sources":["../../../../src/agent-runtime/models/vlm/guiowl.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtD,gFAAgF;AAChF,6BAA6B;AAC7B,EAAE;AACF,yBAAyB;AACzB,sFAAsF;AACtF,uEAAuE;AACvE,EAAE;AACF,wDAAwD;AACxD,gFAAgF;AAChF,4GAA4G;AAC5G,6EAA6E;AAC7E,iFAAiF;AACjF,wFAAwF;AACxF,gFAAgF;AAEhF,MAAM,QAAQ,GAAG,IAAI,CAAC;AACtB,MAAM,QAAQ,GAAG,IAAI,CAAC;AAEtB,SAAS,eAAe,CAAC,CAAS,EAAE,CAAS;IAC3C,MAAM,MAAM,GAAG;QACb,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,cAAc,EAAE,YAAY;YAC5B,IAAI,EAAE,YAAY;YAClB,WAAW,EACT,6EAA6E;gBAC7E,8DAA8D;gBAC9D,gEAAgE;gBAChE,iEAAiE;gBACjE,+FAA+F;gBAC/F,gCAAgC,CAAC,IAAI,CAAC,KAAK;gBAC3C,0EAA0E;gBAC1E,8EAA8E;YAChF,UAAU,EAAE;gBACV,UAAU,EAAE;oBACV,MAAM,EAAE;wBACN,WAAW,EACT,qDAAqD;4BACrD,sDAAsD;4BACtD,gDAAgD;4BAChD,2EAA2E;4BAC3E,oEAAoE;4BACpE,+FAA+F;4BAC/F,gHAAgH;4BAChH,oEAAoE;4BACpE,+CAA+C;4BAC/C,wCAAwC;4BACxC,8DAA8D;4BAC9D,iEAAiE;4BACjE,2EAA2E;4BAC3E,6EAA6E;wBAC/E,IAAI,EAAE;4BACJ,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM;4BAC7C,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW;yBACnE;wBACD,IAAI,EAAE,QAAQ;qBACf;oBACD,UAAU,EAAE;wBACV,WAAW,EACT,8EAA8E;4BAC9E,2EAA2E;wBAC7E,IAAI,EAAE,OAAO;qBACd;oBACD,WAAW,EAAE;wBACX,WAAW,EACT,0DAA0D;wBAC5D,IAAI,EAAE,OAAO;qBACd;oBACD,IAAI,EAAE;wBACJ,WAAW,EACT,sGAAsG;wBACxG,IAAI,EAAE,QAAQ;qBACf;oBACD,IAAI,EAAE;wBACJ,WAAW,EAAE,kEAAkE;wBAC/E,IAAI,EAAE,QAAQ;qBACf;oBACD,MAAM,EAAE;wBACN,WAAW,EACT,qGAAqG;4BACrG,0CAA0C;wBAC5C,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;wBACvC,IAAI,EAAE,QAAQ;qBACf;oBACD,MAAM,EAAE;wBACN,WAAW,EAAE,mDAAmD;wBAChE,IAAI,EAAE,QAAQ;wBACd,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;qBAC7B;iBACF;gBACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;gBACpB,IAAI,EAAE,QAAQ;aACf;YACD,WAAW,EAAE,wCAAwC;SACtD;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,cAAc,CAAC,CAAS,EAAE,CAAS;IAC1C,OAAO;;;;;;EAMP,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC;;;;;;;;;;;;;;;;;;uEAkBgD,CAAC;AACxE,CAAC;AAED,SAAS,cAAc,CAAC,CAAS,EAAE,CAAS;IAC1C,OAAO;;;;;;EAMP,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC;;;;;;;;;;;;;;;;;;wDAkBiC,CAAC;AACzD,CAAC;AAED,SAAS,cAAc,CACrB,IAAY,EACZ,OAAiB,EACjB,IAAY,EACZ,IAAiD;IAEjD,MAAM,IAAI,GAAG,IAAI,KAAK,IAAI,CAAC;IAC3B,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM;QAChD,CAAC,CAAC,CAAC,IAAI;YACL,CAAC,CAAC,kFAAkF;YACpF,CAAC,CAAC,0BAA0B,CAAC;QAC/B,CAAC,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM;YACjC,CAAC,CAAC,CAAC,IAAI;gBACL,CAAC,CAAC,qGAAqG;gBACvG,CAAC,CAAC,iCAAiC,CAAC;YACtC,CAAC,CAAC,CAAC,IAAI;gBACL,CAAC,CAAC,uEAAuE;gBACzE,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAE3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI;YACT,CAAC,CAAC,iCAAiC,OAAO,qBAAqB,IAAI,EAAE;YACrE,CAAC,CAAC,IAAI,OAAO,oBAAoB,IAAI,EAAE,CAAC;IAC5C,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxF,OAAO,IAAI;QACT,CAAC,CAAC,iCAAiC,OAAO,qBAAqB,IAAI,2BAA2B,IAAI,EAAE;QACpG,CAAC,CAAC,IAAI,OAAO,oBAAoB,IAAI,gBAAgB,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,aAAc,SAAQ,YAAY;IAC7B,IAAI,GAAG,QAAQ,CAAC;IACP,kBAAkB,GAAG;QACrC,WAAW,EAAE,QAAQ;QACrB,cAAc,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC;KACnC,CAAC;IACX,4EAA4E;IACnE,QAAQ,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;IAElE,aAAa,GAAa,EAAE,CAAC;IAErC,aAAa,CACX,IAAY,EACZ,UAAsB,EACtB,QAAmB,EACnB,OAAgB,EAChB,IAAY;QAEZ,IAAI,OAAO;YAAE,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QAErC,MAAM,GAAG,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACpG,MAAM,OAAO,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC;QACtC,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC;QAClC,IAAI,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAClG,IAAI,MAAM,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YAC/B,QAAQ,IAAI,IAAI,KAAK,IAAI;gBACvB,CAAC,CAAC,iCAAiC,UAAU,CAAC,KAAK,EAAE;gBACrD,CAAC,CAAC,0BAA0B,UAAU,CAAC,KAAK,EAAE,CAAC;QACnD,CAAC;QAED,MAAM,WAAW,GAAkD;YACjE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE;SACjC,CAAC;QACF,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxB,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,WAAW;gBACjB,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE;aAC5C,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YAChC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,qEAAqE;QACrE,MAAM,eAAe,GAAG,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE3F,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,EAAE,WAAW,CAAC;eACzC,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,0CAA0C,CAAC,CAAC;QAC/E,wEAAwE;QACxE,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC;QAC9B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,aAAa,CAAC,QAAmB,EAAE,SAAwB,EAAE,MAAoB;QAC/E,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACtE,IAAI,IAAI;gBAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QACD,+EAA+E;QAC/E,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,YAAY;QACV,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;IAC1B,CAAC;CACF;AAED,eAAe,CAAC,GAAG,EAAE,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAE3C,SAAS,aAAa,CAAC,MAAoB;IACzC,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC;IAC5C,IAAI,CAAC,KAAK,KAAK;QAAE,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC/C,IAAI,CAAC,KAAK,YAAY;QAAE,OAAO,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC3D,IAAI,CAAC,KAAK,YAAY;QAAE,OAAO,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC3D,IAAI,CAAC,KAAK,MAAM;QAAE,OAAO,SAAS,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC;IAClE,IAAI,CAAC,KAAK,OAAO;QAAE,OAAO,SAAS,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC;IACnF,IAAI,CAAC,KAAK,UAAU;QAAE,OAAO,QAAQ,CAAC,CAAC,QAAQ,GAAG,CAAC;IACnD,IAAI,CAAC,KAAK,YAAY;QAAE,OAAO,qBAAqB,CAAC;IACrD,IAAI,CAAC,KAAK,YAAY;QAAE,OAAO,qBAAqB,CAAC;IACrD,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/agent-runtime/models/vlm/maiui.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/agent-runtime/models/vlm/maiui.js

@@ -1,142 +0,0 @@
-import { invalidModelOutput, ModelAdapter, registerAdapter } from "./base.js";
-import { parseToolCall } from "./shared/tool-call.js";
-// MAIUI adapter
-// Aligned with ClawGUI prompts_maiui.py (MAI_MOBILE_SYS_PROMPT_CN / MAI_MOBILE_SYS_PROMPT_EN)
-//
-// Message structure (mirrors ClawGUI exactly):
-//   First turn  : [system(string), user([text:task]), user([image])]
-//   Subsequent  : [...context, assistant(rawContent), user([image])]
-//   Max 3 images in context window (pruned in updateContext).
-//
-// Output format expected:
-//   <thinking>...</thinking>
-//   <tool_call>{"name": "mobile_use", "arguments": {...}}</tool_call>
-const MAX_IMAGES = 3;
-const SYSTEM_PROMPT_CN = `你是一个 GUI 智能体。你将获得一个任务、你的动作历史以及截图。你需要执行下一个动作来完成任务。
-
-## 输出格式
-对于每个函数调用,在 <thinking> </thinking> 标签中返回思考过程,并在 <tool_call></tool_call> XML 标签中返回包含函数名和参数的 json 对象:
-\`\`\`
-<thinking>
-...
-</thinking>
-<tool_call>
-{"name": "mobile_use", "arguments": <args-json-object>}
-</tool_call>
-\`\`\`
-
-## 动作空间
-
-{"action": "click", "coordinate": [x, y]}
-{"action": "long_press", "coordinate": [x, y]}
-{"action": "type", "text": ""}
-{"action": "swipe", "direction": "up 或 down 或 left 或 right", "coordinate": [x, y]} # "coordinate" 是可选的。如果要滑动特定 UI 元素,请使用 "coordinate"。
-{"action": "open", "text": "app_name"}
-{"action": "drag", "start_coordinate": [x1, y1], "end_coordinate": [x2, y2]}
-{"action": "system_button", "button": "button_name"} # 选项: back, home, menu, enter
-{"action": "wait"}
-{"action": "terminate", "status": "success 或 fail"}
-{"action": "answer", "text": "xxx"} # 在 text 部分使用转义字符 \\'、\\" 和 \\n 以确保我们可以按正常 python 字符串格式解析内容。
-
-## 注意事项
-- 写一个小计划，然后在 <thinking></thinking> 部分用一句话总结你的下一个动作（及其目标元素）。
-- 你可以使用 \`open\` 动作直接打开应用，这是打开应用最快的方式。
-- 你必须严格遵循动作空间，并在 <thinking> </thinking> 和 <tool_call></tool_call> XML 标签中返回正确的 json 对象。
-`;
-const SYSTEM_PROMPT_EN = `You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
-
-## Output Format
-For each function call, return the thinking process in <thinking> </thinking> tags, and a json object with function name and arguments within <tool_call></tool_call> XML tags:
-\`\`\`
-<thinking>
-...
-</thinking>
-<tool_call>
-{"name": "mobile_use", "arguments": <args-json-object>}
-</tool_call>
-\`\`\`
-
-## Action Space
-
-{"action": "click", "coordinate": [x, y]}
-{"action": "long_press", "coordinate": [x, y]}
-{"action": "type", "text": ""}
-{"action": "swipe", "direction": "up or down or left or right", "coordinate": [x, y]} # "coordinate" is optional. Use the "coordinate" if you want to swipe a specific UI element.
-{"action": "open", "text": "app_name"}
-{"action": "drag", "start_coordinate": [x1, y1], "end_coordinate": [x2, y2]}
-{"action": "system_button", "button": "button_name"} # Options: back, home, menu, enter
-{"action": "wait"}
-{"action": "terminate", "status": "success or fail"}
-{"action": "answer", "text": "xxx"} # Use escape characters \\'、\\" and \\n in text part to ensure we can parse the text in normal python string format.
-
-
-## Note
-- Write a small plan and finally summarize your next action (with its target element) in one sentence in <thinking></thinking> part.
-- You can use the \`open\` action to open the app directly, because it is the fastest way to open the app.
-- You must follow the Action Space strictly, and return the correct json object within <thinking> </thinking> and <tool_call></tool_call> XML tags.
-`;
-class MAIUIAdapter extends ModelAdapter {
-    name = "maiui";
-    patterns = [/maiui/, /mai[-_]?ui/, /mai_ui/];
-    buildMessages(task, screenInfo, context, isFirst, lang) {
-        const systemPrompt = lang === "cn" ? SYSTEM_PROMPT_CN : SYSTEM_PROMPT_EN;
-        if (isFirst) {
-            // First turn: [system(string), user([text:task]), user([image])]
-            const msgs = [
-                { role: "system", content: systemPrompt },
-                { role: "user", content: [{ type: "text", text: task }] },
-            ];
-            if (screenInfo.imageRef) {
-                msgs.push({
-                    role: "user",
-                    content: [{ type: "image_url", image_url: { url: screenInfo.imageRef.url } }],
-                });
-            }
-            return msgs;
-        }
-        // Subsequent turns: [...accumulated context, user([image])]
-        const userContent = [];
-        if (screenInfo.imageRef) {
-            userContent.push({ type: "image_url", image_url: { url: screenInfo.imageRef.url } });
-        }
-        return [...context, { role: "user", content: userContent }];
-    }
-    parseResponse(raw) {
-        const thinking = extractThinking(raw);
-        const cleaned = raw.replace(/<thinking>[\s\S]*?<\/thinking>/gi, "").trim();
-        const toolBlock = cleaned.includes("<tool_call>") ? cleaned : `<tool_call>${cleaned}</tool_call>`;
-        return parseToolCall(toolBlock, thinking)
-            ?? invalidModelOutput(this.name, "expected a valid <tool_call> JSON action");
-    }
-    updateContext(context, response, _parsed) {
-        context.push({ role: "assistant", content: response.rawContent });
-        return pruneImages(context, MAX_IMAGES);
-    }
-}
-registerAdapter(() => new MAIUIAdapter());
-// ── Helpers ──────────────────────────────────────────────────────────────────
-function extractThinking(raw) {
-    const m = raw.match(/<thinking>([\s\S]*?)<\/thinking>/i);
-    return m ? m[1].trim() : "";
-}
-function pruneImages(context, keepCount) {
-    let imageCount = 0;
-    for (let i = context.length - 1; i >= 0; i--) {
-        const msg = context[i];
-        if (Array.isArray(msg.content)) {
-            const parts = msg.content;
-            const hasImage = parts.some((p) => p.type === "image_url");
-            if (hasImage) {
-                imageCount++;
-                if (imageCount > keepCount) {
-                    context[i] = {
-                        ...msg,
-                        content: parts.filter((p) => p.type !== "image_url"),
-                    };
-                }
-            }
-        }
-    }
-    return context;
-}
-//# sourceMappingURL=maiui.js.map

package/dist/agent-runtime/models/vlm/maiui.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"maiui.js","sourceRoot":"","sources":["../../../../src/agent-runtime/models/vlm/maiui.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtD,gBAAgB;AAChB,8FAA8F;AAC9F,EAAE;AACF,+CAA+C;AAC/C,qEAAqE;AACrE,qEAAqE;AACrE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6BAA6B;AAC7B,sEAAsE;AAEtE,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8BxB,CAAC;AAEF,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+BxB,CAAC;AAEF,MAAM,YAAa,SAAQ,YAAY;IAC5B,IAAI,GAAG,OAAO,CAAC;IACf,QAAQ,GAAG,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;IAEtD,aAAa,CACX,IAAY,EACZ,UAAsB,EACtB,OAAkB,EAClB,OAAgB,EAChB,IAAY;QAEZ,MAAM,YAAY,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEzE,IAAI,OAAO,EAAE,CAAC;YACZ,iEAAiE;YACjE,MAAM,IAAI,GAAc;gBACtB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;gBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;aAC1D,CAAC;YACF,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACxB,IAAI,CAAC,IAAI,CAAC;oBACR,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC;iBAC9E,CAAC,CAAC;YACL,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,4DAA4D;QAC5D,MAAM,WAAW,GAAkD,EAAE,CAAC;QACtE,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxB,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACvF,CAAC;QACD,OAAO,CAAC,GAAG,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3E,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,OAAO,cAAc,CAAC;QAClG,OAAO,aAAa,CAAC,SAAS,EAAE,QAAQ,CAAC;eACpC,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,0CAA0C,CAAC,CAAC;IACjF,CAAC;IAED,aAAa,CAAC,OAAkB,EAAE,QAAuB,EAAE,OAAqB;QAC9E,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAClE,OAAO,WAAW,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAC1C,CAAC;CACF;AAED,eAAe,CAAC,GAAG,EAAE,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;AAE1C,gFAAgF;AAEhF,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACzD,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,WAAW,CAAC,OAAkB,EAAE,SAAiB;IACxD,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAkC,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;YAC3D,IAAI,QAAQ,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;oBAC3B,OAAO,CAAC,CAAC,CAAC,GAAG;wBACX,GAAG,GAAG;wBACN,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC;qBACrD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/agent-runtime/models/vlm/qwen38.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/agent-runtime/models/vlm/qwen38.js

@@ -1,241 +0,0 @@
-import { invalidModelOutput, ModelAdapter, registerAdapter } from "./base.js";
-import { parseToolCall } from "./shared/tool-call.js";
-const POINT_SCHEMA = {
-    type: "array",
-    items: { type: "integer", minimum: 0, maximum: 999 },
-    minItems: 2,
-    maxItems: 2,
-};
-const ALLOWED_ACTIONS = [
-    "click",
-    "tap",
-    "left_click",
-    "double_tap",
-    "double_click",
-    "doubleclick",
-    "long_press",
-    "swipe",
-    "drag",
-    "scroll",
-    "type",
-    "input_text",
-    "system_button",
-    "keyboard_enter",
-    "back",
-    "navigate_back",
-    "home",
-    "navigate_home",
-    "open_app",
-    "launch_app",
-    "wait",
-    "answer",
-    "terminate",
-];
-const MOBILE_USE_TOOL = {
-    type: "function",
-    function: {
-        name: "mobile_use",
-        description: "Execute exactly one atomic action on the current Android device.",
-        parameters: {
-            type: "object",
-            additionalProperties: false,
-            properties: {
-                action: {
-                    type: "string",
-                    enum: [
-                        "click",
-                        "double_tap",
-                        "long_press",
-                        "swipe",
-                        "drag",
-                        "scroll",
-                        "type",
-                        "system_button",
-                        "open_app",
-                        "wait",
-                        "answer",
-                        "terminate",
-                    ],
-                },
-                coordinate: {
-                    ...POINT_SCHEMA,
-                    description: "Required for click, double_tap and long_press; may anchor scroll.",
-                },
-                coordinate2: { ...POINT_SCHEMA, description: "Required swipe or drag end point." },
-                start_coordinate: { ...POINT_SCHEMA, description: "Alternative swipe or drag start point." },
-                end_coordinate: { ...POINT_SCHEMA, description: "Alternative swipe or drag end point." },
-                text: { type: "string", description: "Required for type and answer." },
-                app_name: { type: "string", description: "Required for open_app." },
-                button: { type: "string", enum: ["Back", "Home", "Menu", "Enter"] },
-                direction: { type: "string", enum: ["up", "down", "left", "right"] },
-                amount: { type: "number", minimum: 0.1, maximum: 5 },
-                time: { type: "number", minimum: 0, maximum: 30 },
-                status: {
-                    type: "string",
-                    enum: ["success", "failure"],
-                    description: "Required for terminate.",
-                },
-            },
-            required: ["action"],
-        },
-    },
-};
-const SYSTEM_PROMPT_EN = `You control an Android phone using the mobile_use tool.
-Inspect the latest screenshot or UI hierarchy and call mobile_use exactly once per turn.
-Coordinates are normalized integers from 0 to 999, with (0,0) at the top-left.
-Use only the actions declared by the tool. Never emit desktop actions such as left_click.
-Before repeating an action, verify from the latest observation whether the previous action worked.
-Use answer when the user requested information. Use terminate when the task is complete or cannot be completed safely.`;
-const SYSTEM_PROMPT_CN = `你通过 mobile_use 工具控制 Android 手机。
-每一轮检查最新截图或 UI 结构，并且只调用一次 mobile_use。
-坐标是 0 到 999 的归一化整数，左上角为 (0,0)。
-只能使用工具声明的动作，不要输出 left_click 等桌面动作。
-重复动作前，先根据最新观察确认上一步是否生效。
-用户要求获取信息时使用 answer；任务完成或无法安全完成时使用 terminate。`;
-class Qwen38Adapter extends ModelAdapter {
-    name = "qwen38";
-    patterns = [/(?:^|[\/_-])qwen3\.8(?:$|[\/_-])/];
-    observationProfile = {
-        defaultMode: "vision",
-        supportedModes: ["vision", "xml", "hybrid"],
-    };
-    requestProfile = {
-        tools: [MOBILE_USE_TOOL],
-        toolChoice: "required",
-        parallelToolCalls: false,
-    };
-    actionHistory = [];
-    buildMessages(task, screenInfo, _context, isFirst, lang) {
-        if (isFirst)
-            this.actionHistory = [];
-        const content = [{
-                type: "text",
-                text: buildUserPrompt(task, screenInfo, this.actionHistory, lang),
-            }];
-        if (screenInfo.imageRef) {
-            content.push({ type: "image_url", image_url: { url: screenInfo.imageRef.url } });
-        }
-        return [
-            { role: "system", content: lang === "cn" ? SYSTEM_PROMPT_CN : SYSTEM_PROMPT_EN },
-            { role: "user", content },
-        ];
-    }
-    parseResponse(raw) {
-        const marker = raw.indexOf("<tool_call>");
-        const thinking = (marker >= 0 ? raw.slice(0, marker) : raw).trim();
-        const parsed = parseToolCall(raw, thinking, {
-            expectedToolName: "mobile_use",
-            allowedActions: ALLOWED_ACTIONS,
-        })
-            ?? invalidModelOutput(this.name, "expected one valid mobile_use tool call");
-        validateAction(parsed);
-        return parsed;
-    }
-    updateContext(_context, _response, parsed) {
-        if (!parsed.shouldFinish)
-            this.actionHistory.push(actionSummary(parsed));
-        return [];
-    }
-    clearHistory() {
-        this.actionHistory = [];
-    }
-}
-registerAdapter(() => new Qwen38Adapter());
-function buildUserPrompt(task, screenInfo, history, lang) {
-    const lines = lang === "cn"
-        ? [`任务：${task}`, `当前应用：${screenInfo.currentApp || "未知"}`]
-        : [`Task: ${task}`, `Current app: ${screenInfo.currentApp || "unknown"}`];
-    if (history.length > 0) {
-        lines.push(lang === "cn" ? "已执行动作：" : "Actions already executed:");
-        history.forEach((action, index) => lines.push(`${index + 1}. ${action}`));
-    }
-    if (screenInfo.uiXml) {
-        lines.push(lang === "cn" ? "UI 结构：" : "UI hierarchy:", screenInfo.uiXml);
-    }
-    return lines.join("\n");
-}
-function validateAction(parsed) {
-    const p = parsed.params;
-    const point = (x, y, label) => {
-        if (!isCoordinate(x) || !isCoordinate(y)) {
-            invalidModelOutput("qwen38", `${label} coordinates must be integers in [0, 999]`);
-        }
-    };
-    switch (parsed.actionType) {
-        case "tap":
-        case "double_tap":
-            point(p.x, p.y, parsed.actionType);
-            break;
-        case "long_press":
-            point(p.x, p.y, parsed.actionType);
-            if (typeof p.duration_ms !== "number"
-                || !Number.isFinite(p.duration_ms)
-                || p.duration_ms < 0
-                || p.duration_ms > 30_000) {
-                invalidModelOutput("qwen38", "long_press time must be in [0, 30]");
-            }
-            break;
-        case "swipe":
-            point(p.start_x, p.start_y, "swipe start");
-            point(p.end_x, p.end_y, "swipe end");
-            break;
-        case "scroll":
-            point(p.x, p.y, "scroll");
-            if (!["up", "down", "left", "right"].includes(String(p.direction))) {
-                invalidModelOutput("qwen38", "scroll direction is invalid");
-            }
-            if (typeof p.amount !== "number"
-                || !Number.isFinite(p.amount)
-                || p.amount < 0.1
-                || p.amount > 5) {
-                invalidModelOutput("qwen38", "scroll amount must be in [0.1, 5]");
-            }
-            break;
-        case "type":
-            if (typeof p.text !== "string" || p.text.length === 0) {
-                invalidModelOutput("qwen38", "type requires non-empty text");
-            }
-            break;
-        case "open_app":
-            if (typeof p.app_name !== "string" || p.app_name.trim().length === 0) {
-                invalidModelOutput("qwen38", "open_app requires app_name");
-            }
-            break;
-        case "wait": {
-            const seconds = p.duration_seconds;
-            if (typeof seconds !== "number" || !Number.isFinite(seconds) || seconds < 0 || seconds > 30) {
-                invalidModelOutput("qwen38", "wait time must be in [0, 30]");
-            }
-            break;
-        }
-        case "press_back":
-        case "press_home":
-        case "press_key":
-        case "finish":
-        case "fail":
-            break;
-        default:
-            invalidModelOutput("qwen38", `unsupported action: ${parsed.actionType}`);
-    }
-}
-function isCoordinate(value) {
-    return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 999;
-}
-function actionSummary(parsed) {
-    const p = parsed.params;
-    if (parsed.actionType === "tap")
-        return `click(${p.x},${p.y})`;
-    if (parsed.actionType === "double_tap")
-        return `double_tap(${p.x},${p.y})`;
-    if (parsed.actionType === "long_press")
-        return `long_press(${p.x},${p.y})`;
-    if (parsed.actionType === "swipe") {
-        return `swipe(${p.start_x},${p.start_y}->${p.end_x},${p.end_y})`;
-    }
-    if (parsed.actionType === "type")
-        return `type(${JSON.stringify(String(p.text).slice(0, 80))})`;
-    if (parsed.actionType === "open_app")
-        return `open_app(${p.app_name})`;
-    return parsed.actionType;
-}
-//# sourceMappingURL=qwen38.js.map