@beeos-ai/device-mcp-server 0.2.3

package/dist/util/ui-xml.js

@@ -0,0 +1,184 @@
+/**
+ * Android uiautomator XML parser + compact LLM-ready formatter.
+ *
+ * Lifted from the legacy `device-agent/src/device/ui-xml.ts`. Used by the
+ * `ui_dump` MCP tool when the caller passes `query` (server-side filtering
+ * keeps the wire payload small).
+ *
+ * Pure data — no IO, no logging, safe to call inside hot paths.
+ */
+/**
+ * Format raw uiautomator XML as a compact one-line-per-node summary suitable
+ * for VLM prompts.
+ */
+export function formatUiXmlToolOutput(xml, query) {
+    const root = parseXmlTree(xml);
+    if (!root)
+        return "Failed to parse UI XML";
+    const nodes = collectNodes(root);
+    const filtered = query ? filterNodes(nodes, query) : nodes;
+    if (filtered.length === 0) {
+        return query ? `No elements found matching "${query}"` : "No interactive elements found";
+    }
+    return filtered.map(formatNode).join("\n");
+}
+/**
+ * Parse XML and return the structured `UiNode` list (for programmatic use,
+ * e.g. ACP `device/ui-tree`).
+ */
+export function searchUiXmlNodes(xml, query) {
+    const root = parseXmlTree(xml);
+    if (!root)
+        return [];
+    const nodes = collectNodes(root);
+    const filtered = query ? filterNodes(nodes, query) : nodes;
+    return filtered.map(treeNodeToUiNode);
+}
+function parseXmlTree(xml) {
+    const re = /<(\/?)node(\s[^>]*)?(\/?)>/g;
+    let m;
+    const dummy = makeNode("");
+    const stack = [dummy];
+    while ((m = re.exec(xml)) !== null) {
+        const isClose = m[1] === "/";
+        const isSelf = m[3] === "/";
+        const attrs = m[2] ?? "";
+        if (isClose) {
+            if (stack.length > 1)
+                stack.pop();
+        }
+        else {
+            const node = makeNode(attrs);
+            const parent = stack[stack.length - 1];
+            node.parent = parent;
+            parent.children.push(node);
+            if (!isSelf)
+                stack.push(node);
+        }
+    }
+    return dummy.children[0] ?? null;
+}
+function makeNode(attrs) {
+    const bounds = attr(attrs, "bounds");
+    return {
+        text: attr(attrs, "text"),
+        contentDesc: attr(attrs, "content-desc"),
+        resourceId: attr(attrs, "resource-id"),
+        className: attr(attrs, "class"),
+        bounds,
+        clickable: attr(attrs, "clickable") === "true",
+        longClickable: attr(attrs, "long-clickable") === "true",
+        scrollable: attr(attrs, "scrollable") === "true",
+        focusable: attr(attrs, "focusable") === "true",
+        enabled: attr(attrs, "enabled") !== "false",
+        center: boundsCenter(bounds),
+        children: [],
+    };
+}
+function attr(attrs, name) {
+    return attrs.match(new RegExp(`${name}="([^"]*)"`, "i"))?.[1] ?? "";
+}
+function boundsCenter(bounds) {
+    const m = bounds.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/);
+    if (!m)
+        return { x: 0, y: 0 };
+    return { x: Math.round((+m[1] + +m[3]) / 2), y: Math.round((+m[2] + +m[4]) / 2) };
+}
+/* -------------------------------------------------------------------------- */
+/* Node selection / filtering / formatting                                    */
+/* -------------------------------------------------------------------------- */
+function isInteractive(n) {
+    return n.enabled && (n.clickable || n.longClickable || n.scrollable || n.focusable);
+}
+function hasInteractiveAncestor(n) {
+    let p = n.parent;
+    while (p) {
+        if (isInteractive(p))
+            return true;
+        p = p.parent;
+    }
+    return false;
+}
+function displayText(n) {
+    return n.text || n.contentDesc;
+}
+function collectNodes(root) {
+    const result = [];
+    function walk(n) {
+        if (!n.enabled)
+            return;
+        if (isInteractive(n)) {
+            result.push(n);
+        }
+        else if (displayText(n) && !hasInteractiveAncestor(n)) {
+            result.push(n);
+        }
+        for (const child of n.children)
+            walk(child);
+    }
+    walk(root);
+    return result;
+}
+function filterNodes(nodes, query) {
+    const q = query.toLowerCase();
+    return nodes.filter((n) => n.text.toLowerCase().includes(q) ||
+        n.contentDesc.toLowerCase().includes(q) ||
+        n.resourceId.toLowerCase().includes(q) ||
+        n.className.toLowerCase().includes(q));
+}
+const CLASS_SHORT = {
+    Button: "Btn",
+    ImageButton: "ImgBtn",
+    TextView: "Text",
+    EditText: "Edit",
+    ImageView: "Img",
+    CheckBox: "Check",
+    Switch: "Switch",
+    RadioButton: "Radio",
+    ListView: "List",
+    RecyclerView: "List",
+    ScrollView: "Scroll",
+    HorizontalScrollView: "HScroll",
+    ViewPager: "Pager",
+    WebView: "Web",
+    FrameLayout: "Frame",
+    LinearLayout: "Layout",
+    RelativeLayout: "Layout",
+    ConstraintLayout: "Layout",
+};
+function shortClass(cls) {
+    const base = cls.split(".").pop() ?? cls;
+    return CLASS_SHORT[base] ?? base;
+}
+function formatNode(n) {
+    const parts = [];
+    parts.push(`(${n.center.x},${n.center.y})`);
+    parts.push(shortClass(n.className));
+    const label = displayText(n);
+    if (label)
+        parts.push(`"${label}"`);
+    if (n.resourceId)
+        parts.push(`id=${n.resourceId}`);
+    if (n.clickable)
+        parts.push("*click");
+    if (n.longClickable)
+        parts.push("*lclick");
+    if (n.scrollable)
+        parts.push("*scroll");
+    if (n.focusable && !n.clickable)
+        parts.push("*focus");
+    return parts.join(" ");
+}
+function treeNodeToUiNode(n) {
+    return {
+        text: n.text,
+        resourceId: n.resourceId,
+        contentDesc: n.contentDesc,
+        className: n.className,
+        bounds: n.bounds,
+        center: n.center,
+        clickable: n.clickable,
+        enabled: n.enabled,
+    };
+}
+//# sourceMappingURL=ui-xml.js.map

package/dist/util/ui-xml.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui-xml.js","sourceRoot":"","sources":["../../src/util/ui-xml.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAW,EAAE,KAAc;IAC/D,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,CAAC,IAAI;QAAE,OAAO,wBAAwB,CAAC;IAE3C,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAE3D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,CAAC,CAAC,+BAA+B,KAAK,GAAG,CAAC,CAAC,CAAC,+BAA+B,CAAC;IAC3F,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,KAAa;IACzD,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC3D,OAAO,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;AACxC,CAAC;AAsBD,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,EAAE,GAAG,6BAA6B,CAAC;IACzC,IAAI,CAAyB,CAAC;IAE9B,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3B,MAAM,KAAK,GAAe,CAAC,KAAK,CAAC,CAAC;IAElC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC;QAC7B,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEzB,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,KAAK,CAAC,GAAG,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;YACrB,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,CAAC,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACnC,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACrC,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,cAAc,CAAC;QACxC,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC;QACtC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC;QAC/B,MAAM;QACN,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,MAAM;QAC9C,aAAa,EAAE,IAAI,CAAC,KAAK,EAAE,gBAAgB,CAAC,KAAK,MAAM;QACvD,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,KAAK,MAAM;QAChD,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,MAAM;QAC9C,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,OAAO;QAC3C,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC;QAC5B,QAAQ,EAAE,EAAE;KACb,CAAC;AACJ,CAAC;AAED,SAAS,IAAI,CAAC,KAAa,EAAE,IAAY;IACvC,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,IAAI,YAAY,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACzD,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IAC9B,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;AACpF,CAAC;AAED,gFAAgF;AAChF,gFAAgF;AAChF,gFAAgF;AAEhF,SAAS,aAAa,CAAC,CAAW;IAChC,OAAO,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,sBAAsB,CAAC,CAAW;IACzC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,CAAC;QACT,IAAI,aAAa,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QAClC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,CAAW;IAC9B,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC;AACjC,CAAC;AAED,SAAS,YAAY,CAAC,IAAc;IAClC,MAAM,MAAM,GAAe,EAAE,CAAC;IAE9B,SAAS,IAAI,CAAC,CAAW;QACvB,IAAI,CAAC,CAAC,CAAC,OAAO;YAAE,OAAO;QAEvB,IAAI,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;aAAM,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,QAAQ;YAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC;IACX,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,KAAiB,EAAE,KAAa;IACnD,MAAM,CAAC,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAC9B,OAAO,KAAK,CAAC,MAAM,CACjB,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChC,CAAC,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;QACvC,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CACxC,CAAC;AACJ,CAAC;AAED,MAAM,WAAW,GAA2B;IAC1C,MAAM,EAAE,KAAK;IACb,WAAW,EAAE,QAAQ;IACrB,QAAQ,EAAE,MAAM;IAChB,QAAQ,EAAE,MAAM;IAChB,SAAS,EAAE,KAAK;IAChB,QAAQ,EAAE,OAAO;IACjB,MAAM,EAAE,QAAQ;IAChB,WAAW,EAAE,OAAO;IACpB,QAAQ,EAAE,MAAM;IAChB,YAAY,EAAE,MAAM;IACpB,UAAU,EAAE,QAAQ;IACpB,oBAAoB,EAAE,SAAS;IAC/B,SAAS,EAAE,OAAO;IAClB,OAAO,EAAE,KAAK;IACd,WAAW,EAAE,OAAO;IACpB,YAAY,EAAE,QAAQ;IACtB,cAAc,EAAE,QAAQ;IACxB,gBAAgB,EAAE,QAAQ;CAC3B,CAAC;AAEF,SAAS,UAAU,CAAC,GAAW;IAC7B,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC;IACzC,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AACnC,CAAC;AAED,SAAS,UAAU,CAAC,CAAW;IAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAEpC,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;IACnD,IAAI,CAAC,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,CAAC,CAAC,aAAa;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3C,IAAI,CAAC,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAEtD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAW;IACnC,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;AACJ,CAAC"}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@beeos-ai/device-mcp-server",
+  "version": "0.2.3",
+  "description": "In-pod HTTP MCP tool server for BeeOS device-agent (Android / desktop / mobile sandboxes).",
+  "license": "Apache-2.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./backends": {
+      "types": "./dist/backends/index.d.ts",
+      "import": "./dist/backends/index.js"
+    },
+    "./server": {
+      "types": "./dist/server/index.d.ts",
+      "import": "./dist/server/index.js"
+    }
+  },
+  "bin": {
+    "device-mcp-server": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "fastify": "^5.8.5",
+    "pino": "^10.3.1",
+    "sharp": "^0.34.5",
+    "zod": "^4.3.6",
+    "@beeos-ai/device-common": "0.2.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.8.0",
+    "vitest": "^4.1.5"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json && chmod +x dist/cli.js",
+    "start": "tsx src/cli.ts",
+    "dev": "tsx --watch src/cli.ts",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}