agent-device 0.2.6 → 0.3.0

package/src/platforms/android/__tests__/index.test.ts

@@ -0,0 +1,74 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { findBounds, parseUiHierarchy } from '../ui-hierarchy.ts';
+
+test('parseUiHierarchy reads double-quoted Android node attributes', () => {
+  const xml =
+    '<hierarchy><node class="android.widget.TextView" text="Hello" content-desc="Greeting" resource-id="com.demo:id/title" bounds="[10,20][110,60]" clickable="true" enabled="true"/></hierarchy>';
+
+  const result = parseUiHierarchy(xml, 800, { raw: true });
+  assert.equal(result.nodes.length, 1);
+  assert.equal(result.nodes[0].value, 'Hello');
+  assert.equal(result.nodes[0].label, 'Hello');
+  assert.equal(result.nodes[0].identifier, 'com.demo:id/title');
+  assert.deepEqual(result.nodes[0].rect, { x: 10, y: 20, width: 100, height: 40 });
+  assert.equal(result.nodes[0].hittable, true);
+  assert.equal(result.nodes[0].enabled, true);
+});
+
+test('parseUiHierarchy reads single-quoted Android node attributes', () => {
+  const xml =
+    "<hierarchy><node class='android.widget.TextView' text='Hello' content-desc='Greeting' resource-id='com.demo:id/title' bounds='[10,20][110,60]' clickable='true' enabled='true'/></hierarchy>";
+
+  const result = parseUiHierarchy(xml, 800, { raw: true });
+  assert.equal(result.nodes.length, 1);
+  assert.equal(result.nodes[0].value, 'Hello');
+  assert.equal(result.nodes[0].label, 'Hello');
+  assert.equal(result.nodes[0].identifier, 'com.demo:id/title');
+  assert.deepEqual(result.nodes[0].rect, { x: 10, y: 20, width: 100, height: 40 });
+  assert.equal(result.nodes[0].hittable, true);
+  assert.equal(result.nodes[0].enabled, true);
+});
+
+test('parseUiHierarchy supports mixed quote styles in one node', () => {
+  const xml =
+    '<hierarchy><node class="android.widget.TextView" text=\'Hello\' content-desc="Greeting" resource-id=\'com.demo:id/title\' bounds="[10,20][110,60]"/></hierarchy>';
+
+  const result = parseUiHierarchy(xml, 800, { raw: true });
+  assert.equal(result.nodes.length, 1);
+  assert.equal(result.nodes[0].value, 'Hello');
+  assert.equal(result.nodes[0].label, 'Hello');
+  assert.equal(result.nodes[0].identifier, 'com.demo:id/title');
+});
+
+test('findBounds supports single and double quoted attributes', () => {
+  const xml = [
+    '<hierarchy>',
+    '<node text="Nothing" content-desc="Irrelevant" bounds="[0,0][10,10]"/>',
+    "<node text='Target from single quote' content-desc='Alt single' bounds='[100,200][300,500]'/>",
+    '<node text="Target from double quote" content-desc="Alt double" bounds="[50,50][150,250]"/>',
+    '</hierarchy>',
+  ].join('');
+
+  assert.deepEqual(findBounds(xml, 'single quote'), { x: 200, y: 350 });
+  assert.deepEqual(findBounds(xml, 'alt double'), { x: 100, y: 150 });
+});
+
+test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
+  const xml =
+    "<hierarchy><node class='android.widget.TextView' hint-text='Spoofed' text='Actual' bounds='[10,20][110,60]'/></hierarchy>";
+
+  const result = parseUiHierarchy(xml, 800, { raw: true });
+  assert.equal(result.nodes.length, 1);
+  assert.equal(result.nodes[0].value, 'Actual');
+});
+
+test('findBounds ignores bounds-like fragments inside other attribute values', () => {
+  const xml = [
+    '<hierarchy>',
+    "<node text='Target' content-desc=\"metadata bounds='[900,900][1000,1000]'\" bounds='[100,200][300,500]'/>",
+    '</hierarchy>',
+  ].join('');
+
+  assert.deepEqual(findBounds(xml, 'target'), { x: 200, y: 350 });
+});

package/src/platforms/android/index.ts

@@ -3,8 +3,9 @@ import { runCmd, whichCmd } from '../../utils/exec.ts';
 import { withRetry } from '../../utils/retry.ts';
 import { AppError } from '../../utils/errors.ts';
 import type { DeviceInfo } from '../../utils/device.ts';
-import type { RawSnapshotNode, Rect, SnapshotOptions } from '../../utils/snapshot.ts';
+import type { RawSnapshotNode, SnapshotOptions } from '../../utils/snapshot.ts';
 import { waitForAndroidBoot } from './devices.ts';
+import { findBounds, parseBounds, parseUiHierarchy, readNodeAttributes } from './ui-hierarchy.ts';
 
 const ALIASES: Record<string, { type: 'intent' | 'package'; value: string }> = {
   settings: { type: 'intent', value: 'android.settings.SETTINGS' },
@@ -449,12 +450,52 @@ async function dumpUiHierarchy(device: DeviceInfo): Promise<string> {
 }
 
 async function dumpUiHierarchyOnce(device: DeviceInfo): Promise<string> {
-  await runCmd(
+  // Preferred: stream XML directly to stdout, avoiding file I/O race conditions.
+  const streamed = await runCmd(
     'adb',
-    adbArgs(device, ['shell', 'uiautomator', 'dump', '/sdcard/window_dump.xml']),
+    adbArgs(device, ['exec-out', 'uiautomator', 'dump', '/dev/tty']),
+    { allowFailure: true },
   );
-  const result = await runCmd('adb', adbArgs(device, ['shell', 'cat', '/sdcard/window_dump.xml']));
-  return result.stdout;
+  if (streamed.exitCode === 0) {
+    const fromStream = extractUiDumpXml(streamed.stdout, streamed.stderr);
+    if (fromStream) return fromStream;
+  }
+
+  // Fallback: dump to file and read back.
+  // If `cat` fails with "no such file", the outer withRetry (via isRetryableAdbError) handles it.
+  const dumpPath = '/sdcard/window_dump.xml';
+  const dumpResult = await runCmd(
+    'adb',
+    adbArgs(device, ['shell', 'uiautomator', 'dump', dumpPath]),
+  );
+  const actualPath = resolveDumpPath(dumpPath, dumpResult.stdout, dumpResult.stderr);
+
+  const result = await runCmd('adb', adbArgs(device, ['shell', 'cat', actualPath]));
+  const xml = extractUiDumpXml(result.stdout, result.stderr);
+  if (!xml) {
+    throw new AppError('COMMAND_FAILED', 'uiautomator dump did not return XML', {
+      stdout: result.stdout,
+      stderr: result.stderr,
+    });
+  }
+  return xml;
+}
+
+function resolveDumpPath(defaultPath: string, stdout: string, stderr: string): string {
+  const text = `${stdout}\n${stderr}`;
+  const match = /dumped to:\s*(\S+)/i.exec(text);
+  return match?.[1] ?? defaultPath;
+}
+
+function extractUiDumpXml(stdout: string, stderr: string): string | null {
+  const text = `${stdout}\n${stderr}`;
+  const start = text.indexOf('<?xml');
+  const hierarchyStart = start >= 0 ? start : text.indexOf('<hierarchy');
+  if (hierarchyStart < 0) return null;
+  const end = text.lastIndexOf('</hierarchy>');
+  if (end < 0 || end < hierarchyStart) return null;
+  const xml = text.slice(hierarchyStart, end + '</hierarchy>'.length).trim();
+  return xml.length > 0 ? xml : null;
 }
 
 function isRetryableAdbError(err: unknown): boolean {
@@ -467,6 +508,7 @@ function isRetryableAdbError(err: unknown): boolean {
   if (stderr.includes('connection reset')) return true;
   if (stderr.includes('broken pipe')) return true;
   if (stderr.includes('timed out')) return true;
+  if (stderr.includes('no such file or directory')) return true;
   return false;
 }
 
@@ -582,291 +624,3 @@ async function sleep(ms: number): Promise<void> {
 function clampCount(value: number, min: number, max: number): number {
   return Math.max(min, Math.min(max, value));
 }
-
-function findBounds(xml: string, query: string): { x: number; y: number } | null {
-  const q = query.toLowerCase();
-  const nodeRegex = /<node[^>]+>/g;
-  let match = nodeRegex.exec(xml);
-  while (match) {
-    const node = match[0];
-    const textMatch = /text="([^"]*)"/.exec(node);
-    const descMatch = /content-desc="([^"]*)"/.exec(node);
-    const textVal = (textMatch?.[1] ?? '').toLowerCase();
-    const descVal = (descMatch?.[1] ?? '').toLowerCase();
-    if (textVal.includes(q) || descVal.includes(q)) {
-      const boundsMatch = /bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"/.exec(node);
-      if (boundsMatch) {
-        const x1 = Number(boundsMatch[1]);
-        const y1 = Number(boundsMatch[2]);
-        const x2 = Number(boundsMatch[3]);
-        const y2 = Number(boundsMatch[4]);
-        return { x: Math.floor((x1 + x2) / 2), y: Math.floor((y1 + y2) / 2) };
-      }
-      return { x: 0, y: 0 };
-    }
-    match = nodeRegex.exec(xml);
-  }
-  return null;
-}
-
-function parseUiHierarchy(
-  xml: string,
-  maxNodes: number,
-  options: SnapshotOptions,
-): { nodes: RawSnapshotNode[]; truncated?: boolean } {
-  const tree = parseUiHierarchyTree(xml);
-  const nodes: RawSnapshotNode[] = [];
-  let truncated = false;
-  const maxDepth = options.depth ?? Number.POSITIVE_INFINITY;
-  const scopedRoot = options.scope ? findScopeNode(tree, options.scope) : null;
-  const roots = scopedRoot ? [scopedRoot] : tree.children;
-
-  const interactiveDescendantMemo = new Map<AndroidNode, boolean>();
-  const hasInteractiveDescendant = (node: AndroidNode): boolean => {
-    const cached = interactiveDescendantMemo.get(node);
-    if (cached !== undefined) return cached;
-    for (const child of node.children) {
-      if (child.hittable || hasInteractiveDescendant(child)) {
-        interactiveDescendantMemo.set(node, true);
-        return true;
-      }
-    }
-    interactiveDescendantMemo.set(node, false);
-    return false;
-  };
-
-  const walk = (
-    node: AndroidNode,
-    depth: number,
-    parentIndex?: number,
-    ancestorHittable: boolean = false,
-    ancestorCollection: boolean = false,
-  ) => {
-    if (nodes.length >= maxNodes) {
-      truncated = true;
-      return;
-    }
-    if (depth > maxDepth) return;
-
-    const include = options.raw
-      ? true
-      : shouldIncludeAndroidNode(
-          node,
-          options,
-          ancestorHittable,
-          hasInteractiveDescendant(node),
-          ancestorCollection,
-        );
-    let currentIndex = parentIndex;
-    if (include) {
-      currentIndex = nodes.length;
-      nodes.push({
-        index: currentIndex,
-        type: node.type ?? undefined,
-        label: node.label ?? undefined,
-        value: node.value ?? undefined,
-        identifier: node.identifier ?? undefined,
-        rect: node.rect,
-        enabled: node.enabled,
-        hittable: node.hittable,
-        depth,
-        parentIndex,
-      });
-    }
-    const nextAncestorHittable = ancestorHittable || Boolean(node.hittable);
-    const nextAncestorCollection = ancestorCollection || isCollectionContainerType(node.type);
-    for (const child of node.children) {
-      walk(child, depth + 1, currentIndex, nextAncestorHittable, nextAncestorCollection);
-      if (truncated) return;
-    }
-  };
-
-  for (const root of roots) {
-    walk(root, 0, undefined, false, false);
-    if (truncated) break;
-  }
-
-  return truncated ? { nodes, truncated } : { nodes };
-}
-
-function readNodeAttributes(node: string): {
-  text: string | null;
-  desc: string | null;
-  resourceId: string | null;
-  className: string | null;
-  bounds: string | null;
-  clickable?: boolean;
-  enabled?: boolean;
-  focusable?: boolean;
-  focused?: boolean;
-} {
-  const getAttr = (name: string): string | null => {
-    const regex = new RegExp(`${name}="([^"]*)"`);
-    const match = regex.exec(node);
-    return match ? match[1] : null;
-  };
-  const boolAttr = (name: string): boolean | undefined => {
-    const raw = getAttr(name);
-    if (raw === null) return undefined;
-    return raw === 'true';
-  };
-  return {
-    text: getAttr('text'),
-    desc: getAttr('content-desc'),
-    resourceId: getAttr('resource-id'),
-    className: getAttr('class'),
-    bounds: getAttr('bounds'),
-    clickable: boolAttr('clickable'),
-    enabled: boolAttr('enabled'),
-    focusable: boolAttr('focusable'),
-    focused: boolAttr('focused'),
-  };
-}
-
-function parseBounds(bounds: string | null): Rect | undefined {
-  if (!bounds) return undefined;
-  const match = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(bounds);
-  if (!match) return undefined;
-  const x1 = Number(match[1]);
-  const y1 = Number(match[2]);
-  const x2 = Number(match[3]);
-  const y2 = Number(match[4]);
-  return { x: x1, y: y1, width: Math.max(0, x2 - x1), height: Math.max(0, y2 - y1) };
-}
-
-type AndroidNode = {
-  type: string | null;
-  label: string | null;
-  value: string | null;
-  identifier: string | null;
-  rect?: Rect;
-  enabled?: boolean;
-  hittable?: boolean;
-  depth: number;
-  parentIndex?: number;
-  children: AndroidNode[];
-};
-
-function parseUiHierarchyTree(xml: string): AndroidNode {
-  const root: AndroidNode = {
-    type: null,
-    label: null,
-    value: null,
-    identifier: null,
-    depth: -1,
-    children: [],
-  };
-  const stack: AndroidNode[] = [root];
-  const tokenRegex = /<node\b[^>]*>|<\/node>/g;
-  let match = tokenRegex.exec(xml);
-  while (match) {
-    const token = match[0];
-    if (token.startsWith('</node')) {
-      if (stack.length > 1) stack.pop();
-      match = tokenRegex.exec(xml);
-      continue;
-    }
-    const attrs = readNodeAttributes(token);
-    const rect = parseBounds(attrs.bounds);
-    const parent = stack[stack.length - 1];
-    const node: AndroidNode = {
-      type: attrs.className,
-      label: attrs.text || attrs.desc,
-      value: attrs.text,
-      identifier: attrs.resourceId,
-      rect,
-      enabled: attrs.enabled,
-      hittable: attrs.clickable ?? attrs.focusable,
-      depth: parent.depth + 1,
-      parentIndex: undefined,
-      children: [],
-    };
-    parent.children.push(node);
-    if (!token.endsWith('/>')) {
-      stack.push(node);
-    }
-    match = tokenRegex.exec(xml);
-  }
-  return root;
-}
-
-function shouldIncludeAndroidNode(
-  node: AndroidNode,
-  options: SnapshotOptions,
-  ancestorHittable: boolean,
-  descendantHittable: boolean,
-  ancestorCollection: boolean,
-): boolean {
-  const type = normalizeAndroidType(node.type);
-  const hasText = Boolean(node.label && node.label.trim().length > 0);
-  const hasId = Boolean(node.identifier && node.identifier.trim().length > 0);
-  const hasMeaningfulText = hasText && !isGenericAndroidId(node.label ?? '');
-  const hasMeaningfulId = hasId && !isGenericAndroidId(node.identifier ?? '');
-  const isStructural = isStructuralAndroidType(type);
-  const isVisual = type === 'imageview' || type === 'imagebutton';
-  if (options.interactiveOnly) {
-    if (node.hittable) return true;
-    // Keep text proxies for tappable rows while dropping structural noise.
-    const proxyCandidate = hasMeaningfulText || hasMeaningfulId;
-    if (!proxyCandidate) return false;
-    if (isVisual) return false;
-    if (isStructural && !ancestorCollection) return false;
-    return ancestorHittable || descendantHittable || ancestorCollection;
-  }
-  if (options.compact) {
-    return hasMeaningfulText || hasMeaningfulId || Boolean(node.hittable);
-  }
-  if (isStructural || isVisual) {
-    if (node.hittable) return true;
-    if (hasMeaningfulText) return true;
-    if (hasMeaningfulId && descendantHittable) return true;
-    return descendantHittable;
-  }
-  return true;
-}
-
-function isCollectionContainerType(type: string | null): boolean {
-  if (!type) return false;
-  const normalized = normalizeAndroidType(type);
-  return (
-    normalized.includes('recyclerview') ||
-    normalized.includes('listview') ||
-    normalized.includes('gridview')
-  );
-}
-
-function normalizeAndroidType(type: string | null): string {
-  if (!type) return '';
-  return type.toLowerCase();
-}
-
-function isStructuralAndroidType(type: string): boolean {
-  const short = type.split('.').pop() ?? type;
-  return (
-    short.includes('layout') ||
-    short === 'viewgroup' ||
-    short === 'view'
-  );
-}
-
-function isGenericAndroidId(value: string): boolean {
-  const trimmed = value.trim();
-  if (!trimmed) return false;
-  return /^[\w.]+:id\/[\w.-]+$/i.test(trimmed);
-}
-
-function findScopeNode(root: AndroidNode, scope: string): AndroidNode | null {
-  const query = scope.toLowerCase();
-  const stack: AndroidNode[] = [...root.children];
-  while (stack.length > 0) {
-    const node = stack.shift() as AndroidNode;
-    const label = node.label?.toLowerCase() ?? '';
-    const value = node.value?.toLowerCase() ?? '';
-    const identifier = node.identifier?.toLowerCase() ?? '';
-    if (label.includes(query) || value.includes(query) || identifier.includes(query)) {
-      return node;
-    }
-    stack.push(...node.children);
-  }
-  return null;
-}