agent-device 0.2.1 → 0.2.3

package/src/platforms/android/index.ts

@@ -148,25 +148,58 @@ function parseAndroidFocus(text: string): { package?: string; activity?: string
   return null;
 }
 
-export async function openAndroidApp(device: DeviceInfo, app: string): Promise<void> {
+export async function openAndroidApp(
+  device: DeviceInfo,
+  app: string,
+  activity?: string,
+): Promise<void> {
   if (!device.booted) {
     await waitForAndroidBoot(device.id);
   }
   const resolved = await resolveAndroidApp(device, app);
   if (resolved.type === 'intent') {
+    if (activity) {
+      throw new AppError('INVALID_ARGS', 'Activity override requires a package name, not an intent');
+    }
     await runCmd('adb', adbArgs(device, ['shell', 'am', 'start', '-a', resolved.value]));
     return;
   }
+  if (activity) {
+    const component = activity.includes('/')
+      ? activity
+      : `${resolved.value}/${activity.startsWith('.') ? activity : `.${activity}`}`;
+    await runCmd(
+      'adb',
+      adbArgs(device, [
+        'shell',
+        'am',
+        'start',
+        '-a',
+        'android.intent.action.MAIN',
+        '-c',
+        'android.intent.category.DEFAULT',
+        '-c',
+        'android.intent.category.LAUNCHER',
+        '-n',
+        component,
+      ]),
+    );
+    return;
+  }
   await runCmd(
     'adb',
     adbArgs(device, [
       'shell',
-      'monkey',
-      '-p',
-      resolved.value,
+      'am',
+      'start',
+      '-a',
+      'android.intent.action.MAIN',
+      '-c',
+      'android.intent.category.DEFAULT',
       '-c',
       'android.intent.category.LAUNCHER',
-      '1',
+      '-p',
+      resolved.value,
     ]),
   );
 }
@@ -242,8 +275,30 @@ export async function fillAndroid(
   y: number,
   text: string,
 ): Promise<void> {
+  const attempts = [
+    { clearPadding: 12, minClear: 8, maxClear: 48, chunkSize: 4, delayMs: 0 },
+    { clearPadding: 24, minClear: 16, maxClear: 96, chunkSize: 1, delayMs: 15 },
+  ] as const;
+
   await focusAndroid(device, x, y);
-  await typeAndroid(device, text);
+  let lastActual: string | null = null;
+
+  for (const attempt of attempts) {
+    const clearCount = clampCount(
+      text.length + attempt.clearPadding,
+      attempt.minClear,
+      attempt.maxClear,
+    );
+    await clearFocusedText(device, clearCount);
+    await typeAndroidChunked(device, text, attempt.chunkSize, attempt.delayMs);
+    lastActual = await readInputValueAtPoint(device, x, y);
+    if (lastActual === text) return;
+  }
+
+  throw new AppError('COMMAND_FAILED', 'Android fill verification failed', {
+    expected: text,
+    actual: lastActual ?? null,
+  });
 }
 
 export async function scrollAndroid(
@@ -422,6 +477,112 @@ function parseSettingState(state: string): boolean {
   throw new AppError('INVALID_ARGS', `Invalid setting state: ${state}`);
 }
 
+async function typeAndroidChunked(
+  device: DeviceInfo,
+  text: string,
+  chunkSize: number,
+  delayMs: number,
+): Promise<void> {
+  const size = Math.max(1, Math.floor(chunkSize));
+  for (let i = 0; i < text.length; i += size) {
+    const chunk = text.slice(i, i + size);
+    await typeAndroid(device, chunk);
+    if (delayMs > 0 && i + size < text.length) {
+      await sleep(delayMs);
+    }
+  }
+}
+
+async function clearFocusedText(device: DeviceInfo, count: number): Promise<void> {
+  const deletes = Math.max(0, count);
+  await runCmd('adb', adbArgs(device, ['shell', 'input', 'keyevent', 'KEYCODE_MOVE_END']), {
+    allowFailure: true,
+  });
+  const batchSize = 24;
+  for (let i = 0; i < deletes; i += batchSize) {
+    const size = Math.min(batchSize, deletes - i);
+    await runCmd(
+      'adb',
+      adbArgs(device, ['shell', 'input', 'keyevent', ...Array(size).fill('KEYCODE_DEL')]),
+      {
+        allowFailure: true,
+      },
+    );
+  }
+}
+
+async function readInputValueAtPoint(
+  device: DeviceInfo,
+  x: number,
+  y: number,
+): Promise<string | null> {
+  const xml = await dumpUiHierarchy(device);
+  const nodeRegex = /<node\b[^>]*>/g;
+  let match: RegExpExecArray | null;
+  let focusedEdit: { text: string; area: number } | null = null;
+  let editAtPoint: { text: string; area: number } | null = null;
+  let anyAtPoint: { text: string; area: number } | null = null;
+
+  while ((match = nodeRegex.exec(xml)) !== null) {
+    const node = match[0];
+    const attrs = readNodeAttributes(node);
+    const rect = parseBounds(attrs.bounds);
+    if (!rect) continue;
+    const className = attrs.className ?? '';
+    const text = decodeXmlEntities(attrs.text ?? '');
+    const focused = attrs.focused ?? false;
+    if (!text) continue;
+    const area = Math.max(1, rect.width * rect.height);
+    const containsPoint =
+      x >= rect.x &&
+      x <= rect.x + rect.width &&
+      y >= rect.y &&
+      y <= rect.y + rect.height;
+
+    if (focused && isEditTextClass(className)) {
+      if (!focusedEdit || area <= focusedEdit.area) {
+        focusedEdit = { text, area };
+      }
+      continue;
+    }
+    if (containsPoint && isEditTextClass(className)) {
+      if (!editAtPoint || area <= editAtPoint.area) {
+        editAtPoint = { text, area };
+      }
+      continue;
+    }
+    if (containsPoint) {
+      if (!anyAtPoint || area <= anyAtPoint.area) {
+        anyAtPoint = { text, area };
+      }
+    }
+  }
+
+  return focusedEdit?.text ?? editAtPoint?.text ?? anyAtPoint?.text ?? null;
+}
+
+function isEditTextClass(className: string): boolean {
+  const lower = className.toLowerCase();
+  return lower.includes('edittext') || lower.includes('textfield');
+}
+
+function decodeXmlEntities(value: string): string {
+  return value
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&amp;/g, '&');
+}
+
+async function sleep(ms: number): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function clampCount(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(max, value));
+}
+
 function findBounds(xml: string, query: string): { x: number; y: number } | null {
   const q = query.toLowerCase();
   const nodeRegex = /<node[^>]+>/g;
@@ -460,17 +621,47 @@ function parseUiHierarchy(
   const scopedRoot = options.scope ? findScopeNode(tree, options.scope) : null;
   const roots = scopedRoot ? [scopedRoot] : tree.children;
 
-  const walk = (node: AndroidNode, depth: number) => {
+  const interactiveDescendantMemo = new Map<AndroidNode, boolean>();
+  const hasInteractiveDescendant = (node: AndroidNode): boolean => {
+    const cached = interactiveDescendantMemo.get(node);
+    if (cached !== undefined) return cached;
+    for (const child of node.children) {
+      if (child.hittable || hasInteractiveDescendant(child)) {
+        interactiveDescendantMemo.set(node, true);
+        return true;
+      }
+    }
+    interactiveDescendantMemo.set(node, false);
+    return false;
+  };
+
+  const walk = (
+    node: AndroidNode,
+    depth: number,
+    parentIndex?: number,
+    ancestorHittable: boolean = false,
+    ancestorCollection: boolean = false,
+  ) => {
     if (nodes.length >= maxNodes) {
       truncated = true;
       return;
     }
     if (depth > maxDepth) return;
 
-    const include = options.raw ? true : shouldIncludeAndroidNode(node, options);
+    const include = options.raw
+      ? true
+      : shouldIncludeAndroidNode(
+          node,
+          options,
+          ancestorHittable,
+          hasInteractiveDescendant(node),
+          ancestorCollection,
+        );
+    let currentIndex = parentIndex;
     if (include) {
+      currentIndex = nodes.length;
       nodes.push({
-        index: nodes.length,
+        index: currentIndex,
         type: node.type ?? undefined,
         label: node.label ?? undefined,
         value: node.value ?? undefined,
@@ -479,17 +670,19 @@ function parseUiHierarchy(
         enabled: node.enabled,
         hittable: node.hittable,
         depth,
-        parentIndex: node.parentIndex,
+        parentIndex,
       });
     }
+    const nextAncestorHittable = ancestorHittable || Boolean(node.hittable);
+    const nextAncestorCollection = ancestorCollection || isCollectionContainerType(node.type);
     for (const child of node.children) {
-      walk(child, depth + 1);
+      walk(child, depth + 1, currentIndex, nextAncestorHittable, nextAncestorCollection);
       if (truncated) return;
     }
   };
 
   for (const root of roots) {
-    walk(root, 0);
+    walk(root, 0, undefined, false, false);
     if (truncated) break;
   }
 
@@ -505,6 +698,7 @@ function readNodeAttributes(node: string): {
   clickable?: boolean;
   enabled?: boolean;
   focusable?: boolean;
+  focused?: boolean;
 } {
   const getAttr = (name: string): string | null => {
     const regex = new RegExp(`${name}="([^"]*)"`);
@@ -525,6 +719,7 @@ function readNodeAttributes(node: string): {
     clickable: boolAttr('clickable'),
     enabled: boolAttr('enabled'),
     focusable: boolAttr('focusable'),
+    focused: boolAttr('focused'),
   };
 }
 
@@ -595,18 +790,71 @@ function parseUiHierarchyTree(xml: string): AndroidNode {
   return root;
 }
 
-function shouldIncludeAndroidNode(node: AndroidNode, options: SnapshotOptions): boolean {
+function shouldIncludeAndroidNode(
+  node: AndroidNode,
+  options: SnapshotOptions,
+  ancestorHittable: boolean,
+  descendantHittable: boolean,
+  ancestorCollection: boolean,
+): boolean {
+  const type = normalizeAndroidType(node.type);
+  const hasText = Boolean(node.label && node.label.trim().length > 0);
+  const hasId = Boolean(node.identifier && node.identifier.trim().length > 0);
+  const hasMeaningfulText = hasText && !isGenericAndroidId(node.label ?? '');
+  const hasMeaningfulId = hasId && !isGenericAndroidId(node.identifier ?? '');
+  const isStructural = isStructuralAndroidType(type);
+  const isVisual = type === 'imageview' || type === 'imagebutton';
   if (options.interactiveOnly) {
-    return Boolean(node.hittable);
+    if (node.hittable) return true;
+    // Keep text proxies for tappable rows while dropping structural noise.
+    const proxyCandidate = hasMeaningfulText || hasMeaningfulId;
+    if (!proxyCandidate) return false;
+    if (isVisual) return false;
+    if (isStructural && !ancestorCollection) return false;
+    return ancestorHittable || descendantHittable || ancestorCollection;
   }
   if (options.compact) {
-    const hasText = Boolean(node.label && node.label.trim().length > 0);
-    const hasId = Boolean(node.identifier && node.identifier.trim().length > 0);
-    return hasText || hasId || Boolean(node.hittable);
+    return hasMeaningfulText || hasMeaningfulId || Boolean(node.hittable);
+  }
+  if (isStructural || isVisual) {
+    if (node.hittable) return true;
+    if (hasMeaningfulText) return true;
+    if (hasMeaningfulId && descendantHittable) return true;
+    return descendantHittable;
   }
   return true;
 }
 
+function isCollectionContainerType(type: string | null): boolean {
+  if (!type) return false;
+  const normalized = normalizeAndroidType(type);
+  return (
+    normalized.includes('recyclerview') ||
+    normalized.includes('listview') ||
+    normalized.includes('gridview')
+  );
+}
+
+function normalizeAndroidType(type: string | null): string {
+  if (!type) return '';
+  return type.toLowerCase();
+}
+
+function isStructuralAndroidType(type: string): boolean {
+  const short = type.split('.').pop() ?? type;
+  return (
+    short.includes('layout') ||
+    short === 'viewgroup' ||
+    short === 'view'
+  );
+}
+
+function isGenericAndroidId(value: string): boolean {
+  const trimmed = value.trim();
+  if (!trimmed) return false;
+  return /^[\w.]+:id\/[\w.-]+$/i.test(trimmed);
+}
+
 function findScopeNode(root: AndroidNode, scope: string): AndroidNode | null {
   const query = scope.toLowerCase();
   const stack: AndroidNode[] = [...root.children];

package/src/platforms/ios/runner-client.ts

@@ -20,6 +20,7 @@ export type RunnerCommand = {
     | 'home'
     | 'appSwitcher'
     | 'alert'
+    | 'pinch'
     | 'shutdown';
   appBundleId?: string;
   text?: string;
@@ -27,11 +28,13 @@ export type RunnerCommand = {
   x?: number;
   y?: number;
   direction?: 'up' | 'down' | 'left' | 'right';
+  scale?: number;
   interactiveOnly?: boolean;
   compact?: boolean;
   depth?: number;
   scope?: string;
   raw?: boolean;
+  clearFirst?: boolean;
 };
 
 export type RunnerSession = {
@@ -173,10 +176,9 @@ async function ensureRunnerSession(
   await ensureBooted(device.id);
   const xctestrun = await ensureXctestrun(device.id, options);
   const port = await getFreePort();
-  const runnerTimeout = process.env.AGENT_DEVICE_RUNNER_TIMEOUT ?? '0';
   const { xctestrunPath, jsonPath } = await prepareXctestrunWithEnv(
     xctestrun,
-    { AGENT_DEVICE_RUNNER_PORT: String(port), AGENT_DEVICE_RUNNER_TIMEOUT: runnerTimeout },
+    { AGENT_DEVICE_RUNNER_PORT: String(port) },
     `session-${device.id}-${port}`,
   );
   const testPromise = runCmdStreaming(
@@ -187,6 +189,8 @@ async function ensureRunnerSession(
       'AgentDeviceRunnerUITests/RunnerTests/testCommand',
       '-parallel-testing-enabled',
       'NO',
+      '-test-timeouts-enabled',
+      'NO',
       '-maximum-concurrent-test-simulator-destinations',
       '1',
       '-xctestrun',
@@ -202,7 +206,7 @@ async function ensureRunnerSession(
         logChunk(chunk, options.logPath, options.traceLogPath, options.verbose);
       },
       allowFailure: true,
-      env: { ...process.env, AGENT_DEVICE_RUNNER_PORT: String(port), AGENT_DEVICE_RUNNER_TIMEOUT: runnerTimeout },
+      env: { ...process.env, AGENT_DEVICE_RUNNER_PORT: String(port) },
     },
   );
 

package/src/utils/args.ts

@@ -20,6 +20,7 @@ export type ParsedArgs = {
     snapshotBackend?: 'ax' | 'xctest';
     appsFilter?: 'launchable' | 'user-installed' | 'all';
     appsMetadata?: boolean;
+    activity?: string;
     noRecord?: boolean;
     recordJson?: boolean;
     help: boolean;
@@ -125,6 +126,9 @@ export function parseArgs(argv: string[]): ParsedArgs {
         case '--session':
           flags.session = value;
           break;
+        case '--activity':
+          flags.activity = value;
+          break;
         default:
           throw new AppError('INVALID_ARGS', `Unknown flag: ${key}`);
       }
@@ -208,6 +212,7 @@ Flags:
   --device <name>                            Device name to target
   --udid <udid>                              iOS device UDID
   --serial <serial>                          Android device serial
+  --activity <component>                     Android activity to launch (package/Activity)
   --out <path>                               Output path for screenshots
   --session <name>                           Named session
   --verbose                                  Stream daemon/runner logs

package/src/utils/interactors.ts

@@ -28,7 +28,7 @@ import {
 } from '../platforms/ios/index.ts';
 
 export type Interactor = {
-  open(app: string): Promise<void>;
+  open(app: string, options?: { activity?: string }): Promise<void>;
   openDevice(): Promise<void>;
   close(app: string): Promise<void>;
   tap(x: number, y: number): Promise<void>;
@@ -45,7 +45,7 @@ export function getInteractor(device: DeviceInfo): Interactor {
   switch (device.platform) {
     case 'android':
       return {
-        open: (app) => openAndroidApp(device, app),
+        open: (app, options) => openAndroidApp(device, app, options?.activity),
         openDevice: () => openAndroidDevice(device),
         close: (app) => closeAndroidApp(device, app),
         tap: (x, y) => pressAndroid(device, x, y),

package/src/utils/output.ts

@@ -28,9 +28,10 @@ type SnapshotNode = {
 
 export function formatSnapshotText(
   data: Record<string, unknown>,
-  options: { raw?: boolean } = {},
+  options: { raw?: boolean; flatten?: boolean } = {},
 ): string {
-  const nodes = (data.nodes ?? []) as SnapshotNode[];
+  const rawNodes = data.nodes;
+  const nodes = Array.isArray(rawNodes) ? (rawNodes as SnapshotNode[]) : [];
   const truncated = Boolean(data.truncated);
   const appName = typeof data.appName === 'string' ? data.appName : undefined;
   const appBundleId = typeof data.appBundleId === 'string' ? data.appBundleId : undefined;
@@ -39,13 +40,17 @@ export function formatSnapshotText(
   if (appBundleId) meta.push(`App: ${appBundleId}`);
   const header = `Snapshot: ${nodes.length} nodes${truncated ? ' (truncated)' : ''}`;
   const prefix = meta.length > 0 ? `${meta.join('\n')}\n` : '';
-  if (!Array.isArray(nodes) || nodes.length === 0) {
+  if (nodes.length === 0) {
     return `${prefix}${header}\n`;
   }
   if (options.raw) {
     const rawLines = nodes.map((node) => JSON.stringify(node));
     return `${prefix}${header}\n${rawLines.join('\n')}\n`;
   }
+  if (options.flatten) {
+    const flatLines = nodes.map((node) => formatSnapshotLine(node, 0, false));
+    return `${prefix}${header}\n${flatLines.join('\n')}\n`;
+  }
   const hiddenGroupDepths: number[] = [];
   const lines: string[] = [];
   for (const node of nodes) {
@@ -62,28 +67,69 @@ export function formatSnapshotText(
     const adjustedDepth = isHiddenGroup
       ? depth
       : Math.max(0, depth - hiddenGroupDepths.length);
-    const indent = '  '.repeat(adjustedDepth);
-    const ref = node.ref ? `@${node.ref}` : '';
-    const flags = [
-      node.enabled === false ? 'disabled' : null,
-    ]
-      .filter(Boolean)
-      .join(', ');
-    const flagText = flags ? ` [${flags}]` : '';
-    const textPart = label ? ` "${label}"` : '';
-    if (isHiddenGroup) {
-      lines.push(`${indent}${ref} [${type}]${flagText}`.trimEnd());
-      continue;
-    }
-    lines.push(`${indent}${ref} [${type}]${textPart}${flagText}`.trimEnd());
+    lines.push(formatSnapshotLine(node, adjustedDepth, isHiddenGroup));
   }
   return `${prefix}${header}\n${lines.join('\n')}\n`;
 }
 
+function formatSnapshotLine(node: SnapshotNode, depth: number, hiddenGroup: boolean): string {
+  const type = formatRole(node.type ?? 'Element');
+  const label = displayLabel(node, type);
+  const indent = '  '.repeat(depth);
+  const ref = node.ref ? `@${node.ref}` : '';
+  const flags = [node.enabled === false ? 'disabled' : null].filter(Boolean).join(', ');
+  const flagText = flags ? ` [${flags}]` : '';
+  const textPart = label ? ` "${label}"` : '';
+  if (hiddenGroup) {
+    return `${indent}${ref} [${type}]${flagText}`.trimEnd();
+  }
+  return `${indent}${ref} [${type}]${textPart}${flagText}`.trimEnd();
+}
+
+function displayLabel(node: SnapshotNode, type: string): string {
+  const label = node.label?.trim();
+  const value = node.value?.trim();
+  if (isEditableRole(type)) {
+    // For inputs, prefer current value over placeholder/accessibility label.
+    if (value) return value;
+    if (label) return label;
+  } else if (label) {
+    return label;
+  }
+  if (value) return value;
+  const identifier = node.identifier?.trim();
+  if (!identifier) return '';
+  if (isGenericResourceId(identifier) && (type === 'group' || type === 'image' || type === 'list' || type === 'collection')) {
+    return '';
+  }
+  return identifier;
+}
+
+function isEditableRole(type: string): boolean {
+  return type === 'text-field' || type === 'text-view' || type === 'search';
+}
+
+function isGenericResourceId(value: string): boolean {
+  return /^[\w.]+:id\/[\w.-]+$/i.test(value);
+}
+
 function formatRole(type: string): string {
+  const raw = type;
   let normalized = type.replace(/XCUIElementType/gi, '').toLowerCase();
-  if (normalized.startsWith("ax")) {
-    normalized = normalized.replace(/^ax/, "");
+  const isAndroidClass =
+    raw.includes('.') &&
+    (raw.startsWith('android.') || raw.startsWith('androidx.') || raw.startsWith('com.'));
+  if (normalized.startsWith('ax')) {
+    normalized = normalized.replace(/^ax/, '');
+  }
+  if (normalized.includes('.')) {
+    normalized = normalized
+      .replace(/^android\.widget\./, '')
+      .replace(/^android\.view\./, '')
+      .replace(/^android\.webkit\./, '')
+      .replace(/^androidx\./, '')
+      .replace(/^com\.google\.android\./, '')
+      .replace(/^com\.android\./, '');
   }
   switch (normalized) {
     case 'application':
@@ -93,24 +139,40 @@ function formatRole(type: string): string {
     case 'tabbar':
       return 'tab-bar';
     case 'button':
+    case 'imagebutton':
       return 'button';
     case 'link':
       return 'link';
     case 'cell':
       return 'cell';
     case 'statictext':
+    case 'checkedtextview':
       return 'text';
     case 'textfield':
+    case 'edittext':
       return 'text-field';
     case 'textview':
+      return isAndroidClass ? 'text' : 'text-view';
+    case 'textarea':
       return 'text-view';
     case 'switch':
       return 'switch';
     case 'slider':
       return 'slider';
     case 'image':
+    case 'imageview':
       return 'image';
-    case 'table':
+    case 'webview':
+      return 'webview';
+    case 'framelayout':
+    case 'linearlayout':
+    case 'relativelayout':
+    case 'constraintlayout':
+    case 'viewgroup':
+    case 'view':
+      return 'group';
+    case 'listview':
+    case 'recyclerview':
       return 'list';
     case 'collectionview':
       return 'collection';
@@ -122,12 +184,6 @@ function formatRole(type: string): string {
       return 'group';
     case 'window':
       return 'window';
-    case 'statictext':
-      return 'text';
-    case 'textfield':
-      return 'text-field';
-    case 'textarea':
-      return 'text-view';
     case 'checkbox':
       return 'checkbox';
     case 'radio':
@@ -137,6 +193,8 @@ function formatRole(type: string): string {
     case 'toolbar':
       return 'toolbar';
     case 'scrollarea':
+    case 'scrollview':
+    case 'nestedscrollview':
       return 'scroll-area';
     case 'table':
       return 'table';