@beeos-ai/device-agent 0.4.0

package/dist/model/adapters/showui.js

@@ -0,0 +1,132 @@
+import { ModelAdapter, registerAdapter } from "./base.js";
+// ShowUI adapter
+// Output format: <point>[[x, y]]</point> for click, or plain text with action keywords
+// Coordinates: 0–1 normalized (fractional), mapped to 0-1000
+const SYSTEM_PROMPT_EN = `\
+You are ShowUI, a mobile GUI automation agent. Given a screenshot and task, output the next action.
+
+For tap/click actions, output the coordinate as:
+<point>[[x, y]]</point>
+where x and y are normalized values between 0 and 1 (e.g., <point>[[0.5, 0.3]]</point>).
+
+For other actions, use one of:
+- TYPE: <text to type>
+- SWIPE: <up|down|left|right>
+- PRESS_BACK
+- PRESS_HOME
+- OPEN_APP: <app name>
+- WAIT
+- FINISH: <reason>
+
+Output one action per turn. Think briefly before acting.`;
+const SYSTEM_PROMPT_CN = `\
+你是 ShowUI，一个手机 GUI 自动化 Agent。根据截图和任务，输出下一步操作。
+
+点击动作，输出坐标：
+<point>[[x, y]]</point>
+x 和 y 为 0–1 之间的归一化值（如 <point>[[0.5, 0.3]]</point>）。
+
+其他动作，使用以下格式：
+- TYPE: <要输入的文本>
+- SWIPE: <up|down|left|right>
+- PRESS_BACK
+- PRESS_HOME
+- OPEN_APP: <应用名称>
+- WAIT
+- FINISH: <原因>
+
+每次只输出一个动作。先简短思考再操作。`;
+class ShowUIAdapter extends ModelAdapter {
+    name = "showui";
+    patterns = [/showui/, /show[-_]?ui/, /show_ui/];
+    buildMessages(task, screenInfo, context, isFirst, lang) {
+        const systemPrompt = lang === "cn" ? SYSTEM_PROMPT_CN : SYSTEM_PROMPT_EN;
+        const msgs = [{ role: "system", content: systemPrompt }];
+        msgs.push(...context);
+        const content = [];
+        if (screenInfo.imageRef) {
+            content.push({ type: "image_url", image_url: { url: screenInfo.imageRef.url } });
+        }
+        const text = isFirst
+            ? `Task: ${task}`
+            : "What is the next action?";
+        content.push({ type: "text", text });
+        msgs.push({ role: "user", content });
+        return msgs;
+    }
+    parseResponse(raw) {
+        return parseShowUIResponse(raw);
+    }
+    updateContext(context, response, _parsed) {
+        // Strip image from last user message before adding assistant reply
+        if (context.length && context[context.length - 1].role === "user") {
+            const last = context[context.length - 1];
+            if (Array.isArray(last.content)) {
+                context[context.length - 1] = {
+                    ...last,
+                    content: last.content.filter((p) => p.type !== "image_url"),
+                };
+            }
+        }
+        context.push({ role: "assistant", content: response.rawContent });
+        return context;
+    }
+}
+registerAdapter(new ShowUIAdapter());
+// ── Parser ────────────────────────────────────────────────────────────────────
+function parseShowUIResponse(raw) {
+    const make = (actionType, params = {}, shouldFinish = false, finishMessage = null) => ({ actionType, params, thinking: "", coordSystem: "0-1000", shouldFinish, finishMessage });
+    // Thinking section
+    const thinkM = raw.match(/<think(?:ing)?>([\s\S]*?)<\/think(?:ing)?>/i);
+    const thinking = thinkM ? thinkM[1].trim() : "";
+    const withThinking = (a) => ({ ...a, thinking });
+    // <point>[[x, y]]</point> — normalized 0-1 coords, convert to 0-1000
+    const pointM = raw.match(/<point>\s*\[\s*\[\s*(\d+(?:\.\d+)?)\s*,\s*(\d+(?:\.\d+)?)\s*\]\s*\]<\/point>/);
+    if (pointM) {
+        const x = Math.round(parseFloat(pointM[1]) * 1000);
+        const y = Math.round(parseFloat(pointM[2]) * 1000);
+        return withThinking(make("tap", { x, y }));
+    }
+    // Also handle single bracket: <point>[x, y]</point>
+    const pointM2 = raw.match(/<point>\s*\[\s*(\d+(?:\.\d+)?)\s*,\s*(\d+(?:\.\d+)?)\s*\]<\/point>/);
+    if (pointM2) {
+        const x = Math.round(parseFloat(pointM2[1]) * 1000);
+        const y = Math.round(parseFloat(pointM2[2]) * 1000);
+        return withThinking(make("tap", { x, y }));
+    }
+    const upper = raw.toUpperCase();
+    // FINISH
+    const finishM = raw.match(/FINISH\s*:\s*(.*)/i);
+    if (finishM) {
+        const msg = finishM[1].trim();
+        return withThinking(make("finish", { message: msg }, true, msg));
+    }
+    // TYPE
+    const typeM = raw.match(/TYPE\s*:\s*([\s\S]+)/i);
+    if (typeM)
+        return withThinking(make("type", { text: typeM[1].trim() }));
+    // SWIPE
+    const swipeM = raw.match(/SWIPE\s*:\s*(\w+)/i);
+    if (swipeM) {
+        const dir = swipeM[1].toLowerCase();
+        const swipeMap = {
+            up: { start_x: 500, start_y: 700, end_x: 500, end_y: 300 },
+            down: { start_x: 500, start_y: 300, end_x: 500, end_y: 700 },
+            left: { start_x: 700, start_y: 500, end_x: 300, end_y: 500 },
+            right: { start_x: 300, start_y: 500, end_x: 700, end_y: 500 },
+        };
+        return withThinking(make("swipe", swipeMap[dir] ?? swipeMap.down));
+    }
+    // OPEN_APP
+    const appM = raw.match(/OPEN_APP\s*:\s*(.*)/i);
+    if (appM)
+        return withThinking(make("open_app", { app_name: appM[1].trim() }));
+    if (upper.includes("PRESS_BACK"))
+        return withThinking(make("press_back"));
+    if (upper.includes("PRESS_HOME"))
+        return withThinking(make("press_home"));
+    if (upper.includes("WAIT"))
+        return withThinking(make("wait", { duration_seconds: 2 }));
+    return withThinking(make("wait", { duration_seconds: 2 }));
+}
+//# sourceMappingURL=showui.js.map

package/dist/model/adapters/showui.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"showui.js","sourceRoot":"","sources":["../../../src/model/adapters/showui.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE1D,iBAAiB;AACjB,uFAAuF;AACvF,6DAA6D;AAE7D,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;yDAgBgC,CAAC;AAE1D,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;oBAgBL,CAAC;AAErB,MAAM,aAAc,SAAQ,YAAY;IAC7B,IAAI,GAAG,QAAQ,CAAC;IAChB,QAAQ,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;IAEzD,aAAa,CACX,IAAY,EACZ,UAAsB,EACtB,OAAkB,EAClB,OAAgB,EAChB,IAAY;QAEZ,MAAM,YAAY,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEzE,MAAM,IAAI,GAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;QACpE,IAAI,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QAEtB,MAAM,OAAO,GAAkD,EAAE,CAAC;QAClE,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,MAAM,IAAI,GAAG,OAAO;YAClB,CAAC,CAAC,SAAS,IAAI,EAAE;YACjB,CAAC,CAAC,0BAA0B,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QAErC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,aAAa,CAAC,OAAkB,EAAE,QAAuB,EAAE,OAAqB;QAC9E,mEAAmE;QACnE,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAClE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACzC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG;oBAC5B,GAAG,IAAI;oBACP,OAAO,EAAG,IAAI,CAAC,OAAmC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC;iBACzF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,eAAe,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAErC,iFAAiF;AAEjF,SAAS,mBAAmB,CAAC,GAAW;IACtC,MAAM,IAAI,GAAG,CACX,UAAkB,EAClB,SAAkC,EAAE,EACpC,YAAY,GAAG,KAAK,EACpB,gBAA+B,IAAI,EACrB,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,CAAC;IAE9G,mBAAmB;IACnB,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAChD,MAAM,YAAY,GAAG,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE/D,qEAAqE;IACrE,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACzG,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACnD,OAAO,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,oDAAoD;IACpD,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;IAChG,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACpD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACpD,OAAO,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAEhC,SAAS;IACT,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,OAAO;IACP,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACjD,IAAI,KAAK;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IAExE,QAAQ;IACR,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,QAAQ,GAA2C;YACvD,EAAE,EAAK,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;YAC7D,IAAI,EAAG,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;YAC7D,IAAI,EAAG,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;YAC7D,KAAK,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;SAC9D,CAAC;QACF,OAAO,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,CAAC;IAED,WAAW;IACX,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC/C,IAAI,IAAI;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IAE9E,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEvF,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC7D,CAAC"}

package/dist/model/adapters/tool-call.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Shared <tool_call> parser used by QwenVL, MAIUI, and GUIOwl adapters.
+ */
+import type { ParsedAction } from "../../types.js";
+export declare function parseToolCall(block: string, thinking: string): ParsedAction | null;
+export declare function argsToAction(args: Record<string, unknown>, thinking: string): ParsedAction;

package/dist/model/adapters/tool-call.js

@@ -0,0 +1,102 @@
+/**
+ * Shared <tool_call> parser used by QwenVL, MAIUI, and GUIOwl adapters.
+ */
+export function parseToolCall(block, thinking) {
+    const m = block.match(/<tool_call>\s*([\s\S]*?)(?:<\/tool_call>|$)/);
+    if (!m)
+        return null;
+    let raw = m[1].trim().replace(/\\n/g, "");
+    // Strip partial closing tag at end
+    raw = raw.replace(/<\/?tool_call[^>]*$/, "").trim();
+    let data;
+    try {
+        data = JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+    const args = (data && typeof data === "object" && "arguments" in data)
+        ? data.arguments
+        : data;
+    if (!args || typeof args !== "object")
+        return null;
+    return argsToAction(args, thinking);
+}
+export function argsToAction(args, thinking) {
+    const action = String(args.action ?? "wait");
+    const coord = args.coordinate;
+    // GUI-Owl / GUI-Plus uses `coordinate2` for swipe endpoint;
+    // older adapters used `end_coordinate`.
+    const startCoord = args.start_coordinate;
+    const endCoord = (args.end_coordinate ?? args.coordinate2);
+    const FINISH_ACTIONS = new Set(["terminate", "answer", "finish", "finished"]);
+    if (FINISH_ACTIONS.has(action)) {
+        const msg = String(args.text ?? args.message ?? args.content ?? args.status ?? "completed");
+        return done("finish", { message: msg }, thinking, true, msg);
+    }
+    if ((action === "tap" || action === "click") && coord) {
+        return done("tap", { x: coord[0], y: coord[1] }, thinking);
+    }
+    if ((action === "double_tap" || action === "doubleClick") && coord) {
+        return done("double_tap", { x: coord[0], y: coord[1] }, thinking);
+    }
+    if (action === "long_press" && coord) {
+        const time = Number(args.time ?? 3);
+        return done("long_press", { x: coord[0], y: coord[1], duration_ms: Math.round(time * 1000) }, thinking);
+    }
+    if (action === "swipe" && coord && endCoord) {
+        return done("swipe", { start_x: coord[0], start_y: coord[1], end_x: endCoord[0], end_y: endCoord[1] }, thinking);
+    }
+    if (action === "swipe" && startCoord && endCoord) {
+        return done("swipe", { start_x: startCoord[0], start_y: startCoord[1], end_x: endCoord[0], end_y: endCoord[1] }, thinking);
+    }
+    if (action === "swipe" && coord) {
+        const dir = String(args.direction ?? "down");
+        const distMap = { short: 150, medium: 300, long: 500 };
+        const dist = distMap[String(args.distance ?? "medium")] ?? 300;
+        const [sx, sy] = [coord[0], coord[1]];
+        const [ex, ey] = dir === "up" ? [sx, sy + dist] :
+            dir === "down" ? [sx, sy - dist] :
+                dir === "left" ? [sx + dist, sy] :
+                    [sx - dist, sy];
+        return done("swipe", { start_x: sx, start_y: sy, end_x: ex, end_y: ey }, thinking);
+    }
+    if (action === "type") {
+        return done("type", { text: String(args.text ?? "") }, thinking);
+    }
+    if (action === "open_app" || action === "open") {
+        return done("open_app", { app_name: String(args.text ?? args.app_name ?? "") }, thinking);
+    }
+    if (action === "press_back" || action === "back") {
+        return done("press_back", {}, thinking);
+    }
+    if (action === "press_home" || action === "home") {
+        return done("press_home", {}, thinking);
+    }
+    if (action === "system_button") {
+        const btn = String(args.button ?? "").toUpperCase();
+        if (btn === "BACK")
+            return done("press_back", {}, thinking);
+        if (btn === "HOME")
+            return done("press_home", {}, thinking);
+        if (btn === "ENTER")
+            return done("press_key", { key: "KEYCODE_ENTER" }, thinking);
+        if (btn === "MENU")
+            return done("press_key", { key: "KEYCODE_MENU" }, thinking);
+        return done("wait", { duration_seconds: 1 }, thinking);
+    }
+    if (action === "key") {
+        // ADB keyevent syntax (e.g. "volume_up" → KEYCODE_VOLUME_UP)
+        const key = String(args.text ?? args.key ?? "").trim();
+        return done("press_key", { key: key.startsWith("KEYCODE_") ? key : `KEYCODE_${key.toUpperCase()}` }, thinking);
+    }
+    if (action === "wait") {
+        const time = Number(args.time ?? args.duration_seconds ?? 2);
+        return done("wait", { duration_seconds: time }, thinking);
+    }
+    return done("wait", { duration_seconds: 2 }, thinking);
+}
+function done(actionType, params, thinking, shouldFinish = false, finishMessage = null) {
+    return { actionType, params, thinking, coordSystem: "0-1000", shouldFinish, finishMessage };
+}
+//# sourceMappingURL=tool-call.js.map

package/dist/model/adapters/tool-call.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-call.js","sourceRoot":"","sources":["../../../src/model/adapters/tool-call.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,QAAgB;IAC3D,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACrE,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpB,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1C,mCAAmC;IACnC,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAEpD,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GACR,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,WAAW,IAAK,IAAe,CAAC;QACnE,CAAC,CAAE,IAA+C,CAAC,SAAS;QAC5D,CAAC,CAAE,IAAgC,CAAC;IAExC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACnD,OAAO,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAA6B,EAAE,QAAgB;IAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,UAAkC,CAAC;IACtD,4DAA4D;IAC5D,wCAAwC;IACxC,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAwC,CAAC;IACjE,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,CAAyB,CAAC;IAEnF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;IAC9E,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,MAAM,CAChB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,IAAI,WAAW,CACxE,CAAC;QACF,OAAO,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;QACtD,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,CAAC,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,aAAa,CAAC,IAAI,KAAK,EAAE,CAAC;QACnE,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,MAAM,KAAK,YAAY,IAAI,KAAK,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC1G,CAAC;IACD,IAAI,MAAM,KAAK,OAAO,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC5C,OAAO,IAAI,CACT,OAAO,EACP,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,EAChF,QAAQ,CACT,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,KAAK,OAAO,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;QACjD,OAAO,IAAI,CACT,OAAO,EACP,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,EAC1F,QAAQ,CACT,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,KAAK,OAAO,IAAI,KAAK,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC;QAC7C,MAAM,OAAO,GAA2B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QAC/E,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC,IAAI,GAAG,CAAC;QAC/D,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GACZ,GAAG,KAAK,IAAI,CAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;YAClC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;gBAClC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;oBACjB,CAAC,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC5F,CAAC;IACD,IAAI,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,MAAM,KAAK,eAAe,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACpD,IAAI,GAAG,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC5D,IAAI,GAAG,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC5D,IAAI,GAAG,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,eAAe,EAAE,EAAE,QAAQ,CAAC,CAAC;QAClF,IAAI,GAAG,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE,QAAQ,CAAC,CAAC;QAChF,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACrB,6DAA6D;QAC7D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,WAAW,EAAE,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IACjH,CAAC;IACD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC;QAC7D,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,IAAI,CACX,UAAkB,EAClB,MAA+B,EAC/B,QAAgB,EAChB,YAAY,GAAG,KAAK,EACpB,gBAA+B,IAAI;IAEnC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9F,CAAC"}

package/dist/model/adapters/uitars.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/model/adapters/uitars.js

@@ -0,0 +1,180 @@
+import { ModelAdapter, registerAdapter } from "./base.js";
+// UITars / Doubao-1.5-UI-TARS adapter
+// Output format: "Thought: ...\nAction: click(point='<point>x y</point>')"
+// Coordinate system: smart_resize (normalized to IMAGE_FACTOR=28 grid)
+const MAX_IMAGES = 5;
+const SYSTEM_PROMPT_EN = `\
+You are a GUI agent. You receive a screenshot and decide the next action.
+
+Output format (always two lines):
+Thought: <brief reasoning>
+Action: <one action call>
+
+Available actions:
+- click(point='<point>x y</point>')
+- long_press(point='<point>x y</point>')
+- double_click(point='<point>x y</point>')
+- type(content='text')
+- scroll(point='<point>x y</point>', direction='up|down|left|right', dist='short|medium|long')
+- press_back()
+- press_home()
+- open_app(app_name='App')
+- wait()
+- finish(message='...')
+
+Coordinates are in smart_resize space (relative to resized image grid).`;
+const SYSTEM_PROMPT_CN = `\
+你是一个 GUI 自动化 Agent。你会收到截图，决定下一步操作。
+
+输出格式（固定两行）：
+Thought: <简短推理>
+Action: <一个动作调用>
+
+可用动作：
+- click(point='<point>x y</point>')
+- long_press(point='<point>x y</point>')
+- double_click(point='<point>x y</point>')
+- type(content='text')
+- scroll(point='<point>x y</point>', direction='up|down|left|right', dist='short|medium|long')
+- press_back()
+- press_home()
+- open_app(app_name='App')
+- wait()
+- finish(message='...')
+
+坐标为 smart_resize 空间（相对于缩放后的图像网格）。`;
+class UITarsAdapter extends ModelAdapter {
+    name = "uitars";
+    patterns = [/uitars/, /ui[-_]?tars/, /doubao.*ui/, /ui.*tars/];
+    // Sliding window of recent image URLs (data:* or signed https). Kept
+    // for parity with the reference agent's MAX_IMAGES window — currently
+    // not replayed into the prompt because the live `context` already
+    // carries vision parts; future turn-rewinding logic will read this.
+    imageHistory = [];
+    buildMessages(task, screenInfo, context, isFirst, lang) {
+        const systemPrompt = lang === "cn" ? SYSTEM_PROMPT_CN : SYSTEM_PROMPT_EN;
+        if (screenInfo.imageRef) {
+            this.imageHistory.push(screenInfo.imageRef.url);
+            if (this.imageHistory.length > MAX_IMAGES) {
+                this.imageHistory.shift();
+            }
+        }
+        const msgs = [{ role: "system", content: systemPrompt }];
+        if (isFirst) {
+            const content = [];
+            if (screenInfo.imageRef) {
+                content.push({ type: "image_url", image_url: { url: screenInfo.imageRef.url } });
+            }
+            content.push({ type: "text", text: `Task: ${task}` });
+            msgs.push({ role: "user", content });
+        }
+        else {
+            // Subsequent turns: rebuild with sliding window of images
+            // Replay context without images, then add current screenshot
+            const textOnlyContext = context.map((m) => {
+                if (Array.isArray(m.content)) {
+                    const textParts = m.content
+                        .filter((p) => p.type === "text");
+                    const stripped = textParts.length === 1 && typeof textParts[0].text === "string"
+                        ? textParts[0].text
+                        : textParts;
+                    return { ...m, content: stripped };
+                }
+                return m;
+            });
+            msgs.push(...textOnlyContext);
+            const content = [];
+            if (screenInfo.imageRef) {
+                content.push({ type: "image_url", image_url: { url: screenInfo.imageRef.url } });
+            }
+            content.push({ type: "text", text: "Current screen. What is the next action?" });
+            msgs.push({ role: "user", content });
+        }
+        return msgs;
+    }
+    parseResponse(raw) {
+        const [thinking, actionStr] = splitThoughtAction(raw);
+        return parseUITarsAction(actionStr, thinking);
+    }
+    updateContext(context, response, parsed) {
+        context.push({ role: "assistant", content: response.rawContent });
+        return context;
+    }
+    clearHistory() {
+        this.imageHistory = [];
+    }
+}
+registerAdapter(new UITarsAdapter());
+// ── Helpers ──────────────────────────────────────────────────────────────────
+function splitThoughtAction(raw) {
+    const thoughtM = raw.match(/Thought\s*:\s*([\s\S]*?)(?=Action\s*:|$)/i);
+    const actionM = raw.match(/Action\s*:\s*([\s\S]*?)$/i);
+    const thinking = thoughtM ? thoughtM[1].trim() : "";
+    const action = actionM ? actionM[1].trim() : raw.trim();
+    return [thinking, action];
+}
+function parsePoint(s) {
+    // <point>x y</point> or plain "x y"
+    const m = s.match(/<point>\s*(\d+(?:\.\d+)?)\s+(\d+(?:\.\d+)?)\s*<\/point>/) ??
+        s.match(/(\d+(?:\.\d+)?)\s+(\d+(?:\.\d+)?)/);
+    if (!m)
+        return null;
+    return [parseFloat(m[1]), parseFloat(m[2])];
+}
+function parseKeyword(s, key) {
+    const m = s.match(new RegExp(`${key}\\s*=\\s*['"]([^'"]*?)['"]`));
+    return m ? m[1] : null;
+}
+function parseUITarsAction(s, thinking) {
+    const make = (actionType, params, shouldFinish = false, finishMessage = null) => ({ actionType, params, thinking, coordSystem: "smart_resize", shouldFinish, finishMessage });
+    const trimmed = s.replace(/^```[a-z]*\s*/i, "").replace(/```$/, "").trim();
+    if (trimmed.startsWith("finish")) {
+        const msg = parseKeyword(trimmed, "message") ?? trimmed;
+        return make("finish", { message: msg }, true, msg);
+    }
+    if (trimmed.startsWith("click")) {
+        const pointStr = parseKeyword(trimmed, "point") ?? trimmed;
+        const pt = parsePoint(pointStr);
+        if (pt)
+            return make("tap", { x: pt[0], y: pt[1] });
+    }
+    if (trimmed.startsWith("long_press")) {
+        const pointStr = parseKeyword(trimmed, "point") ?? trimmed;
+        const pt = parsePoint(pointStr);
+        if (pt)
+            return make("long_press", { x: pt[0], y: pt[1] });
+    }
+    if (trimmed.startsWith("double_click")) {
+        const pointStr = parseKeyword(trimmed, "point") ?? trimmed;
+        const pt = parsePoint(pointStr);
+        if (pt)
+            return make("double_tap", { x: pt[0], y: pt[1] });
+    }
+    if (trimmed.startsWith("type")) {
+        const text = parseKeyword(trimmed, "content") ?? parseKeyword(trimmed, "text") ?? "";
+        return make("type", { text });
+    }
+    if (trimmed.startsWith("scroll")) {
+        const pointStr = parseKeyword(trimmed, "point") ?? "";
+        const pt = parsePoint(pointStr) ?? [500, 500];
+        const dir = parseKeyword(trimmed, "direction") ?? "down";
+        const distStr = parseKeyword(trimmed, "dist") ?? "medium";
+        const distMap = { short: 200, medium: 400, long: 700 };
+        const dist = distMap[distStr] ?? 400;
+        const [sx, sy] = [pt[0], pt[1]];
+        const [ex, ey] = dir === "up" ? [sx, sy - dist] : dir === "down" ? [sx, sy + dist] : dir === "left" ? [sx - dist, sy] : [sx + dist, sy];
+        return make("swipe", { start_x: sx, start_y: sy, end_x: ex, end_y: ey });
+    }
+    if (trimmed.startsWith("press_back"))
+        return make("press_back", {});
+    if (trimmed.startsWith("press_home"))
+        return make("press_home", {});
+    if (trimmed.startsWith("open_app")) {
+        const appName = parseKeyword(trimmed, "app_name") ?? parseKeyword(trimmed, "app") ?? "";
+        return make("open_app", { app_name: appName });
+    }
+    if (trimmed.startsWith("wait"))
+        return make("wait", { duration_seconds: 2 });
+    return make("wait", { duration_seconds: 2 });
+}
+//# sourceMappingURL=uitars.js.map

package/dist/model/adapters/uitars.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"uitars.js","sourceRoot":"","sources":["../../../src/model/adapters/uitars.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE1D,sCAAsC;AACtC,2EAA2E;AAC3E,uEAAuE;AAEvE,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;wEAmB+C,CAAC;AAEzE,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;kCAmBS,CAAC;AAEnC,MAAM,aAAc,SAAQ,YAAY;IAC7B,IAAI,GAAG,QAAQ,CAAC;IAChB,QAAQ,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;IAExE,qEAAqE;IACrE,sEAAsE;IACtE,kEAAkE;IAClE,oEAAoE;IAC5D,YAAY,GAAa,EAAE,CAAC;IAEpC,aAAa,CACX,IAAY,EACZ,UAAsB,EACtB,OAAkB,EAClB,OAAgB,EAChB,IAAY;QAEZ,MAAM,YAAY,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEzE,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;QAEpE,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,GAAkD,EAAE,CAAC;YAClE,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,IAAI,EAAE,EAAE,CAAC,CAAC;YACtD,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,0DAA0D;YAC1D,6DAA6D;YAC7D,MAAM,eAAe,GAAc,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACnD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAI,CAAC,CAAC,OAAyD;yBAC3E,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;oBACpC,MAAM,QAAQ,GACZ,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ;wBAC7D,CAAC,CAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAe;wBAC/B,CAAC,CAAC,SAAS,CAAC;oBAChB,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;gBACrC,CAAC;gBACD,OAAO,CAAC,CAAC;YACX,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAE9B,MAAM,OAAO,GAAkD,EAAE,CAAC;YAClE,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,0CAA0C,EAAE,CAAC,CAAC;YACjF,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACtD,OAAO,iBAAiB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChD,CAAC;IAED,aAAa,CAAC,OAAkB,EAAE,QAAuB,EAAE,MAAoB;QAC7E,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,YAAY;QACV,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;IACzB,CAAC;CACF;AAED,eAAe,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAErC,gFAAgF;AAEhF,SAAS,kBAAkB,CAAC,GAAW;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IACxD,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,oCAAoC;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,yDAAyD,CAAC;QAClE,CAAC,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,YAAY,CAAC,CAAS,EAAE,GAAW;IAC1C,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,4BAA4B,CAAC,CAAC,CAAC;IAClE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzB,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAS,EAAE,QAAgB;IACpD,MAAM,IAAI,GAAG,CAAC,UAAkB,EAAE,MAA+B,EAAE,YAAY,GAAG,KAAK,EAAE,gBAA+B,IAAI,EAAgB,EAAE,CAC5I,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,CAAC;IAE/F,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAE3E,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC;QACxD,OAAO,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC;QAC3D,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QAChC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC;QAC3D,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QAChC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC;QAC3D,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QAChC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;QACrF,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;QACtD,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC;QACzD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC;QAC1D,MAAM,OAAO,GAA2B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QAC/E,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;QACrC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC;QACxI,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACpE,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAEpE,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;QACxF,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC;IAE7E,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC;AAC/C,CAAC"}

package/dist/model/base.d.ts

@@ -0,0 +1,32 @@
+import type { ScreenInfo, ParsedAction, ModelResponse } from "../types.js";
+export type Message = {
+    role: "system" | "user" | "assistant";
+    content: string | Array<{
+        type: string;
+        [k: string]: unknown;
+    }>;
+};
+export interface ModelRequestOptions {
+    verbose?: boolean;
+    signal?: AbortSignal;
+    /**
+     * Streaming token callback. Adapters that buffer tokens internally
+     * (to detect action markers) MAY emit pre-marker text here in
+     * larger chunks; that's fine — `Agent.run` only cares about
+     * forwarding raw deltas to its `onText` hook.
+     */
+    onTextDelta?: (delta: string) => void | Promise<void>;
+}
+export interface ModelClient {
+    readonly modelName: string;
+    readonly adapterName: string;
+    buildMessages(task: string, screenInfo: ScreenInfo, context: Message[], isFirst: boolean, lang: string): Message[];
+    /**
+     * Issue a model request. The legacy positional `(verbose, signal)`
+     * signature is retained for backward compatibility; new callers should
+     * pass an `ModelRequestOptions` bag to also receive streaming deltas.
+     */
+    request(messages: Message[], verboseOrOptions?: boolean | ModelRequestOptions, signal?: AbortSignal): Promise<ModelResponse>;
+    parseResponse(raw: string): ParsedAction;
+    updateContext(context: Message[], response: ModelResponse, parsed: ParsedAction): Message[];
+}

package/dist/model/base.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=base.js.map

package/dist/model/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/model/base.ts"],"names":[],"mappings":""}

package/dist/model/client.d.ts

@@ -0,0 +1,26 @@
+import type { ScreenInfo, ParsedAction, ModelResponse } from "../types.js";
+import type { ModelClient, Message, ModelRequestOptions } from "./base.js";
+import type { ModelAdapter } from "./adapters/base.js";
+import "./adapters/index.js";
+export interface ModelConfig {
+    baseUrl?: string;
+    apiKey?: string;
+    modelName?: string;
+    maxTokens?: number;
+    temperature?: number;
+    topP?: number;
+}
+export declare class OpenAIClient implements ModelClient {
+    private readonly openai;
+    private readonly config;
+    private readonly adapter;
+    constructor(opts?: ModelConfig & {
+        adapter?: string | ModelAdapter;
+    });
+    get modelName(): string;
+    get adapterName(): string;
+    buildMessages(task: string, screenInfo: ScreenInfo, context: Message[], isFirst: boolean, lang: string): Message[];
+    request(messages: Message[], verboseOrOptions?: boolean | ModelRequestOptions, signalArg?: AbortSignal): Promise<ModelResponse>;
+    parseResponse(raw: string): ParsedAction;
+    updateContext(context: Message[], response: ModelResponse, parsed: ParsedAction): Message[];
+}