@phone-use/sdk 0.4.1 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@phone-use/sdk",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "Typed mobile-device SDK — Device, backends, errors, actions",
   "type": "module",
   "license": "Apache-2.0",

package/src/backends/android-hierarchy.ts

@@ -0,0 +1,153 @@
+// Parse an Android `uiautomator dump` XML tree into phone-use SnapshotNodes.
+//
+// The dump is a flat-ish XML of <node> elements, each carrying the attributes
+// Android exposes for accessibility: text, content-desc, resource-id, class,
+// bounds, and the interactive flags (clickable, enabled, focused, …). This is
+// the Android analogue of the iOS accessibility tree the other backends read.
+
+import type { Rect, SnapshotNode } from '../device.ts';
+
+/** `[x1,y1][x2,y2]` → a top-left Rect. Returns undefined if unparseable. */
+export function parseBounds(bounds: string | undefined): Rect | undefined {
+  const m = bounds?.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
+  if (!m) return undefined;
+  const x1 = Number(m[1]);
+  const y1 = Number(m[2]);
+  const x2 = Number(m[3]);
+  const y2 = Number(m[4]);
+  return { x: x1, y: y1, width: x2 - x1, height: y2 - y1 };
+}
+
+/** `android.widget.Button` → `Button`; keeps a bare token as-is. */
+function shortType(className: string | undefined): string | undefined {
+  if (!className) return undefined;
+  const tail = className.split('.').pop();
+  return tail && tail.length > 0 ? tail : className;
+}
+
+/**
+ * Normalise Android widget classes to the harness's canonical (iOS-named)
+ * element types, so every layer above — inputFields(), setField(), the
+ * interactive/editable sets in the SDK — works unchanged on Android.
+ *
+ * This is load-bearing: without it `type --field` reports "no editable text
+ * field" on a real Android because `AutoCompleteTextView` never matches the
+ * SDK's EDITABLE set ({SearchField, TextField, SecureTextField}). Measured on
+ * a OnePlus Nord: the Settings search box is an AutoCompleteTextView.
+ */
+export function canonicalType(
+  shortName: string | undefined,
+  attrs: Record<string, string>,
+): string | undefined {
+  if (!shortName) return undefined;
+  const isPassword = attrs.password === 'true';
+  switch (shortName) {
+    case 'EditText':
+    case 'AutoCompleteTextView':
+    case 'MultiAutoCompleteTextView':
+    case 'SearchView':
+    case 'TextInputEditText':
+      return isPassword ? 'SecureTextField' : 'TextField';
+    case 'Switch':
+    case 'SwitchCompat':
+    case 'CheckBox':
+    case 'ToggleButton':
+    case 'RadioButton':
+      return 'Switch';
+    case 'SeekBar':
+      return 'Slider';
+    case 'ImageButton':
+      return 'Button';
+    default:
+      // Any clickable container is a tap target regardless of its widget
+      // class. DeskClock's tab bar is clickable FrameLayouts with content-desc
+      // ('Stopwatch', 'Timer'…): they rendered in observe but FrameLayout is
+      // not a TAPPABLE role, so `tap "Stopwatch"` failed 16 straight runs on a
+      // control that was on screen. Clickable → Button makes every such
+      // control targetable by label.
+      if (attrs.clickable === 'true') return 'Button';
+      return shortName;
+  }
+}
+
+const ATTR = /([\w:-]+)="([^"]*)"/g;
+
+function attrs(fragment: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const m of fragment.matchAll(ATTR)) {
+    const key = m[1];
+    const val = m[2];
+    if (key !== undefined && val !== undefined) out[key] = decodeEntities(val);
+  }
+  return out;
+}
+
+function decodeEntities(s: string): string {
+  return s
+    .replaceAll('&amp;', '&')
+    .replaceAll('&lt;', '<')
+    .replaceAll('&gt;', '>')
+    .replaceAll('&quot;', '"')
+    .replaceAll('&apos;', "'");
+}
+
+const isTrue = (v: string | undefined): boolean => v === 'true';
+
+export type ParsedHierarchy = {
+  nodes: SnapshotNode[];
+  /** ref → rect, so a later press-by-ref can tap the element's centre. */
+  rects: Map<string, Rect>;
+  /** The package the tree belongs to (frontmost app), if the dump carries one. */
+  packageName?: string;
+};
+
+/**
+ * Parse a uiautomator XML dump. Each <node> becomes a SnapshotNode with a
+ * fresh `@aN` ref. When `interactiveOnly`, keep only nodes that are actually
+ * actionable (clickable / long-clickable / a focusable field) or that carry a
+ * label — the rest is layout scaffolding an agent should not try to tap.
+ */
+export function parseHierarchy(xml: string, opts: { interactiveOnly?: boolean } = {}): ParsedHierarchy {
+  const nodes: SnapshotNode[] = [];
+  const rects = new Map<string, Rect>();
+  let packageName: string | undefined;
+  let index = 0;
+
+  // Match both self-closing <node ... /> and opening <node ...> tags; the tree
+  // is walked flat, which is enough — geometry lives in bounds, not nesting.
+  for (const m of xml.matchAll(/<node\b([^>]*?)\/?>/g)) {
+    const fragment = m[1];
+    if (fragment === undefined) continue;
+    const a = attrs(fragment);
+    if (!packageName && a.package) packageName = a.package;
+
+    const label = a.text || a['content-desc'] || undefined;
+    const clickable = isTrue(a.clickable) || isTrue(a['long-clickable']);
+    const focusableField = isTrue(a.focusable) && (label !== undefined || isTrue(a.editable));
+
+    if (opts.interactiveOnly && !clickable && !focusableField && !label) continue;
+
+    index += 1;
+    const ref = `@a${index}`;
+    const rect = parseBounds(a.bounds);
+    if (rect) rects.set(ref, rect);
+
+    const node: SnapshotNode = {
+      ref,
+      type: canonicalType(shortType(a.class), a),
+      role: canonicalType(shortType(a.class), a),
+      ...(label !== undefined ? { label } : {}),
+      ...(a.text && a.text !== label ? { value: a.text } : {}),
+      ...(a['resource-id'] ? { identifier: a['resource-id'] } : {}),
+      enabled: a.enabled ? isTrue(a.enabled) : true,
+      selected: isTrue(a.selected),
+      focused: isTrue(a.focused),
+      ...(rect ? { rect } : {}),
+      // A disabled control can't be actioned; surface why, mirroring iOS.
+      ...(a.enabled && !isTrue(a.enabled) ? { interactionBlocked: 'disabled' } : {}),
+    };
+    nodes.push(node);
+  }
+
+  return { nodes, rects, ...(packageName ? { packageName } : {}) };
+}