@sensaiorg/adapter-android 0.1.0

package/src/util/ui-tree-parser.test.ts

@@ -0,0 +1,182 @@
+import { describe, it, expect } from "vitest";
+import { parseUiTree, flattenTree } from "./ui-tree-parser.js";
+
+const MINIMAL_XML = `<?xml version="1.0" encoding="UTF-8"?>
+<hierarchy rotation="0">
+  <node index="0" text="Hello" resource-id="com.example:id/greeting"
+        class="android.widget.TextView" package="com.example"
+        content-desc="Greeting text" checkable="false" checked="false"
+        clickable="true" enabled="true" focusable="true" focused="false"
+        scrollable="false" long-clickable="false" password="false"
+        selected="false" bounds="[0,0][500,100]">
+  </node>
+</hierarchy>`;
+
+const NESTED_XML = `<?xml version="1.0" encoding="UTF-8"?>
+<hierarchy rotation="0">
+  <node index="0" text="" resource-id="" class="android.widget.FrameLayout"
+        package="com.example" content-desc="" checkable="false" checked="false"
+        clickable="false" enabled="true" focusable="false" focused="false"
+        scrollable="false" long-clickable="false" password="false"
+        selected="false" bounds="[0,0][1080,1920]">
+    <node index="0" text="Button 1" resource-id="com.example:id/btn1"
+          class="android.widget.Button" package="com.example"
+          content-desc="First button" checkable="false" checked="false"
+          clickable="true" enabled="true" focusable="true" focused="false"
+          scrollable="false" long-clickable="false" password="false"
+          selected="false" bounds="[50,100][300,200]">
+    </node>
+    <node index="1" text="Button 2" resource-id="com.example:id/btn2"
+          class="android.widget.Button" package="com.example"
+          content-desc="" checkable="false" checked="false"
+          clickable="true" enabled="true" focusable="true" focused="false"
+          scrollable="false" long-clickable="false" password="false"
+          selected="false" bounds="[50,250][300,350]">
+    </node>
+  </node>
+</hierarchy>`;
+
+const SYSTEM_UI_XML = `<?xml version="1.0" encoding="UTF-8"?>
+<hierarchy rotation="0">
+  <node index="0" text="12:00" resource-id="" class="android.widget.TextView"
+        package="com.android.systemui" content-desc="" checkable="false" checked="false"
+        clickable="false" enabled="true" focusable="false" focused="false"
+        scrollable="false" long-clickable="false" password="false"
+        selected="false" bounds="[0,0][100,50]">
+  </node>
+  <node index="1" text="App Content" resource-id="" class="android.widget.TextView"
+        package="com.example" content-desc="" checkable="false" checked="false"
+        clickable="false" enabled="true" focusable="false" focused="false"
+        scrollable="false" long-clickable="false" password="false"
+        selected="false" bounds="[0,50][1080,1920]">
+  </node>
+</hierarchy>`;
+
+describe("parseUiTree", () => {
+  it("parses minimal XML with one node", () => {
+    const nodes = parseUiTree(MINIMAL_XML);
+    expect(nodes).toHaveLength(1);
+
+    const node = nodes[0];
+    expect(node.text).toBe("Hello");
+    expect(node.resourceId).toBe("com.example:id/greeting");
+    expect(node.className).toBe("android.widget.TextView");
+    expect(node.contentDescription).toBe("Greeting text");
+    expect(node.clickable).toBe(true);
+    expect(node.enabled).toBe(true);
+    expect(node.focusable).toBe(true);
+    expect(node.scrollable).toBe(false);
+    expect(node.checked).toBe(false);
+    expect(node.selected).toBe(false);
+    expect(node.packageName).toBe("com.example");
+    expect(node.depth).toBe(0);
+    expect(node.bounds).toEqual({ left: 0, top: 0, right: 500, bottom: 100 });
+    expect(node.children).toHaveLength(0);
+  });
+
+  it("parses nested nodes", () => {
+    const nodes = parseUiTree(NESTED_XML);
+    expect(nodes).toHaveLength(1);
+    expect(nodes[0].children).toHaveLength(2);
+    expect(nodes[0].children[0].text).toBe("Button 1");
+    expect(nodes[0].children[1].text).toBe("Button 2");
+    expect(nodes[0].children[0].depth).toBe(1);
+  });
+
+  it("filters system UI by default", () => {
+    const nodes = parseUiTree(SYSTEM_UI_XML);
+    expect(nodes).toHaveLength(1);
+    expect(nodes[0].text).toBe("App Content");
+  });
+
+  it("includes system UI when requested", () => {
+    const nodes = parseUiTree(SYSTEM_UI_XML, { includeSystemUI: true });
+    expect(nodes).toHaveLength(2);
+  });
+
+  it("respects maxDepth", () => {
+    const nodes = parseUiTree(NESTED_XML, { maxDepth: 1 });
+    expect(nodes).toHaveLength(1);
+    // maxDepth=1 means depth 0 is included, children at depth 1 are included but not their children
+    expect(nodes[0].children).toHaveLength(2);
+  });
+
+  it("maxDepth=0 means unlimited", () => {
+    const nodes = parseUiTree(NESTED_XML, { maxDepth: 0 });
+    expect(nodes[0].children).toHaveLength(2);
+  });
+
+  it("returns empty array for empty hierarchy", () => {
+    const xml = `<?xml version="1.0" encoding="UTF-8"?><hierarchy rotation="0"></hierarchy>`;
+    expect(parseUiTree(xml)).toEqual([]);
+  });
+
+  it("returns empty array for missing hierarchy", () => {
+    const xml = `<?xml version="1.0" encoding="UTF-8"?><root></root>`;
+    expect(parseUiTree(xml)).toEqual([]);
+  });
+
+  it("returns empty array for non-XML input", () => {
+    // fast-xml-parser is lenient — doesn't throw on all malformed input
+    const nodes = parseUiTree("not xml at all");
+    expect(nodes).toEqual([]);
+  });
+
+  it("handles node with no bounds", () => {
+    const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<hierarchy rotation="0">
+  <node index="0" text="NoBounds" resource-id="" class="android.view.View"
+        package="com.example" content-desc="" checkable="false" checked="false"
+        clickable="false" enabled="true" focusable="false" focused="false"
+        scrollable="false" long-clickable="false" password="false"
+        selected="false" bounds="">
+  </node>
+</hierarchy>`;
+    const nodes = parseUiTree(xml);
+    expect(nodes).toHaveLength(1);
+    expect(nodes[0].bounds).toBeNull();
+    expect(nodes[0].visible).toBe(false);
+  });
+
+  it("filters invisible nodes with visibleOnly", () => {
+    const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<hierarchy rotation="0">
+  <node index="0" text="Visible" resource-id="" class="android.view.View"
+        package="com.example" content-desc="" checkable="false" checked="false"
+        clickable="false" enabled="true" focusable="false" focused="false"
+        scrollable="false" long-clickable="false" password="false"
+        selected="false" bounds="[0,0][100,100]">
+  </node>
+  <node index="1" text="Invisible" resource-id="" class="android.view.View"
+        package="com.example" content-desc="" checkable="false" checked="false"
+        clickable="false" enabled="true" focusable="false" focused="false"
+        scrollable="false" long-clickable="false" password="false"
+        selected="false" bounds="">
+  </node>
+</hierarchy>`;
+    const nodes = parseUiTree(xml, { visibleOnly: true });
+    expect(nodes).toHaveLength(1);
+    expect(nodes[0].text).toBe("Visible");
+  });
+});
+
+describe("flattenTree", () => {
+  it("flattens nested tree to array", () => {
+    const nodes = parseUiTree(NESTED_XML);
+    const flat = flattenTree(nodes);
+    expect(flat).toHaveLength(3); // root + 2 children
+    expect(flat[0].className).toBe("android.widget.FrameLayout");
+    expect(flat[1].text).toBe("Button 1");
+    expect(flat[2].text).toBe("Button 2");
+  });
+
+  it("handles empty array", () => {
+    expect(flattenTree([])).toEqual([]);
+  });
+
+  it("handles single node", () => {
+    const nodes = parseUiTree(MINIMAL_XML);
+    const flat = flattenTree(nodes);
+    expect(flat).toHaveLength(1);
+  });
+});

package/src/util/ui-tree-parser.ts

@@ -0,0 +1,169 @@
+/**
+ * UI Tree Parser - Converts Android uiautomator XML dump into structured JSON.
+ *
+ * The XML comes from `adb shell uiautomator dump` and represents the full
+ * view hierarchy of the screen.
+ */
+
+import { XMLParser } from "fast-xml-parser";
+
+/** A single node in the parsed UI tree. */
+export interface UiNode {
+  className: string;
+  resourceId: string;
+  text: string;
+  contentDescription: string;
+  bounds: { left: number; top: number; right: number; bottom: number } | null;
+  clickable: boolean;
+  focusable: boolean;
+  enabled: boolean;
+  visible: boolean;
+  scrollable: boolean;
+  checked: boolean;
+  selected: boolean;
+  packageName: string;
+  depth: number;
+  children: UiNode[];
+}
+
+/** Options for controlling the parse output. */
+export interface ParseOptions {
+  /** Maximum depth to traverse (0 = unlimited). */
+  maxDepth?: number;
+  /** Only include nodes that are on-screen. */
+  visibleOnly?: boolean;
+  /** Include system UI nodes (status bar, nav bar). */
+  includeSystemUI?: boolean;
+}
+
+/** System UI packages to filter out. */
+const SYSTEM_UI_PACKAGES = new Set([
+  "com.android.systemui",
+  "com.android.launcher",
+  "com.android.launcher3",
+]);
+
+/**
+ * Parse a uiautomator XML dump into a structured UiNode tree.
+ */
+export function parseUiTree(xml: string, options: ParseOptions = {}): UiNode[] {
+  const { maxDepth = 0, visibleOnly = false, includeSystemUI = false } = options;
+
+  const parser = new XMLParser({
+    ignoreAttributes: false,
+    attributeNamePrefix: "@_",
+    isArray: (name) => name === "node",
+  });
+
+  let parsed: Record<string, unknown>;
+  try {
+    parsed = parser.parse(xml) as Record<string, unknown>;
+  } catch (err) {
+    throw new Error(`Failed to parse UI tree XML: ${err instanceof Error ? err.message : String(err)}`);
+  }
+
+  const hierarchy = parsed["hierarchy"] as Record<string, unknown> | undefined;
+  if (!hierarchy) {
+    return [];
+  }
+
+  const rootNodes = hierarchy["node"];
+  if (!rootNodes) {
+    return [];
+  }
+
+  const nodes = Array.isArray(rootNodes) ? rootNodes : [rootNodes];
+  return nodes
+    .map((n) => convertNode(n as Record<string, unknown>, 0, maxDepth, visibleOnly, includeSystemUI))
+    .filter((n): n is UiNode => n !== null);
+}
+
+/**
+ * Recursively convert a raw XML node object into a UiNode.
+ */
+function convertNode(
+  raw: Record<string, unknown>,
+  depth: number,
+  maxDepth: number,
+  visibleOnly: boolean,
+  includeSystemUI: boolean,
+): UiNode | null {
+  const attr = (key: string): string => String(raw[`@_${key}`] ?? "");
+  const boolAttr = (key: string): boolean => attr(key) === "true";
+
+  const packageName = attr("package");
+
+  // Filter system UI
+  if (!includeSystemUI && SYSTEM_UI_PACKAGES.has(packageName)) {
+    return null;
+  }
+
+  const bounds = parseBounds(attr("bounds"));
+  const visible = bounds !== null && bounds.right > bounds.left && bounds.bottom > bounds.top;
+
+  if (visibleOnly && !visible) {
+    return null;
+  }
+
+  // Recurse into children (respecting maxDepth)
+  let children: UiNode[] = [];
+  if (maxDepth === 0 || depth < maxDepth) {
+    const rawChildren = raw["node"];
+    if (rawChildren) {
+      const childArray = Array.isArray(rawChildren) ? rawChildren : [rawChildren];
+      children = childArray
+        .map((c) => convertNode(c as Record<string, unknown>, depth + 1, maxDepth, visibleOnly, includeSystemUI))
+        .filter((c): c is UiNode => c !== null);
+    }
+  }
+
+  return {
+    className: attr("class"),
+    resourceId: attr("resource-id"),
+    text: attr("text"),
+    contentDescription: attr("content-desc"),
+    bounds,
+    clickable: boolAttr("clickable"),
+    focusable: boolAttr("focusable"),
+    enabled: boolAttr("enabled"),
+    visible,
+    scrollable: boolAttr("scrollable"),
+    checked: boolAttr("checked"),
+    selected: boolAttr("selected"),
+    packageName,
+    depth,
+    children,
+  };
+}
+
+/**
+ * Parse Android bounds string "[left,top][right,bottom]" into an object.
+ */
+function parseBounds(boundsStr: string): UiNode["bounds"] {
+  const match = boundsStr.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/);
+  if (!match) return null;
+
+  return {
+    left: parseInt(match[1], 10),
+    top: parseInt(match[2], 10),
+    right: parseInt(match[3], 10),
+    bottom: parseInt(match[4], 10),
+  };
+}
+
+/**
+ * Flatten the UI tree into an array of nodes (pre-order traversal).
+ */
+export function flattenTree(nodes: UiNode[]): UiNode[] {
+  const result: UiNode[] = [];
+  const stack = [...nodes];
+  while (stack.length > 0) {
+    const node = stack.pop()!;
+    result.push(node);
+    // Push children in reverse so first child is processed first
+    for (let i = node.children.length - 1; i >= 0; i--) {
+      stack.push(node.children[i]);
+    }
+  }
+  return result;
+}

package/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src/**/*.ts"],
+  "references": [
+    { "path": "../core" }
+  ]
+}