agent-device 0.1.0

package/src/utils/device.ts

@@ -0,0 +1,84 @@
+import { AppError } from './errors.ts';
+import { isInteractive } from './interactive.ts';
+import { isCancel, select } from '@clack/prompts';
+
+export type Platform = 'ios' | 'android';
+export type DeviceKind = 'simulator' | 'emulator' | 'device';
+
+export type DeviceInfo = {
+  platform: Platform;
+  id: string;
+  name: string;
+  kind: DeviceKind;
+  booted?: boolean;
+};
+
+export type DeviceSelector = {
+  platform?: Platform;
+  deviceName?: string;
+  udid?: string;
+  serial?: string;
+};
+
+export async function selectDevice(
+  devices: DeviceInfo[],
+  selector: DeviceSelector,
+): Promise<DeviceInfo> {
+  let candidates = devices;
+  const normalize = (value: string): string =>
+    value.toLowerCase().replace(/_/g, ' ').replace(/\s+/g, ' ').trim();
+
+  if (selector.platform) {
+    candidates = candidates.filter((d) => d.platform === selector.platform);
+  }
+
+  if (selector.udid) {
+    const match = candidates.find((d) => d.id === selector.udid && d.platform === 'ios');
+    if (!match) throw new AppError('DEVICE_NOT_FOUND', `No iOS device with UDID ${selector.udid}`);
+    return match;
+  }
+
+  if (selector.serial) {
+    const match = candidates.find((d) => d.id === selector.serial && d.platform === 'android');
+    if (!match)
+      throw new AppError('DEVICE_NOT_FOUND', `No Android device with serial ${selector.serial}`);
+    return match;
+  }
+
+  if (selector.deviceName) {
+    const target = normalize(selector.deviceName);
+    const match = candidates.find((d) => normalize(d.name) === target);
+    if (!match) {
+      throw new AppError('DEVICE_NOT_FOUND', `No device named ${selector.deviceName}`);
+    }
+    return match;
+  }
+
+  if (candidates.length === 1) return candidates[0];
+
+  if (candidates.length === 0) {
+    throw new AppError('DEVICE_NOT_FOUND', 'No devices found', { selector });
+  }
+
+  const booted = candidates.filter((d) => d.booted);
+  if (booted.length === 1) return booted[0];
+
+  if (isInteractive()) {
+    const choice = await select({
+      message: 'Multiple devices available. Choose a device to continue:',
+      options: (booted.length > 0 ? booted : candidates).map((device) => ({
+        label: `${device.name} (${device.platform}${device.kind ? `, ${device.kind}` : ''}${device.booted ? ', booted' : ''})`,
+        value: device.id,
+      })),
+    });
+    if (isCancel(choice)) {
+      throw new AppError('INVALID_ARGS', 'Device selection cancelled');
+    }
+    if (choice) {
+      const match = candidates.find((d) => d.id === choice);
+      if (match) return match;
+    }
+  }
+
+  return booted[0] ?? candidates[0];
+}

package/src/utils/errors.ts

@@ -0,0 +1,35 @@
+export type ErrorCode =
+  | 'INVALID_ARGS'
+  | 'DEVICE_NOT_FOUND'
+  | 'TOOL_MISSING'
+  | 'APP_NOT_INSTALLED'
+  | 'UNSUPPORTED_PLATFORM'
+  | 'UNSUPPORTED_OPERATION'
+  | 'COMMAND_FAILED'
+  | 'UNKNOWN';
+
+export class AppError extends Error {
+  code: ErrorCode;
+  details?: Record<string, unknown>;
+  cause?: unknown;
+
+  constructor(
+    code: ErrorCode,
+    message: string,
+    details?: Record<string, unknown>,
+    cause?: unknown,
+  ) {
+    super(message);
+    this.code = code;
+    this.details = details;
+    this.cause = cause;
+  }
+}
+
+export function asAppError(err: unknown): AppError {
+  if (err instanceof AppError) return err;
+  if (err instanceof Error) {
+    return new AppError('UNKNOWN', err.message, undefined, err);
+  }
+  return new AppError('UNKNOWN', 'Unknown error', { err });
+}

package/src/utils/exec.ts

@@ -0,0 +1,229 @@
+import { spawn, spawnSync } from 'node:child_process';
+import { AppError } from './errors.ts';
+
+export type ExecResult = {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  stdoutBuffer?: Buffer;
+};
+
+export type ExecOptions = {
+  cwd?: string;
+  env?: NodeJS.ProcessEnv;
+  allowFailure?: boolean;
+  binaryStdout?: boolean;
+};
+
+export type ExecStreamOptions = ExecOptions & {
+  onStdoutChunk?: (chunk: string) => void;
+  onStderrChunk?: (chunk: string) => void;
+};
+
+export async function runCmd(
+  cmd: string,
+  args: string[],
+  options: ExecOptions = {},
+): Promise<ExecResult> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, {
+      cwd: options.cwd,
+      env: options.env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+
+    let stdout = '';
+    let stdoutBuffer: Buffer | undefined = options.binaryStdout ? Buffer.alloc(0) : undefined;
+    let stderr = '';
+
+    if (!options.binaryStdout) child.stdout.setEncoding('utf8');
+    child.stderr.setEncoding('utf8');
+
+    child.stdout.on('data', (chunk) => {
+      if (options.binaryStdout) {
+        stdoutBuffer = Buffer.concat([
+          stdoutBuffer ?? Buffer.alloc(0),
+          Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk),
+        ]);
+      } else {
+        stdout += chunk;
+      }
+    });
+
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+
+    child.on('error', (err) => {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === 'ENOENT') {
+        reject(new AppError('TOOL_MISSING', `${cmd} not found in PATH`, { cmd }, err));
+        return;
+      }
+      reject(new AppError('COMMAND_FAILED', `Failed to run ${cmd}`, { cmd, args }, err));
+    });
+
+    child.on('close', (code) => {
+      const exitCode = code ?? 1;
+      if (exitCode !== 0 && !options.allowFailure) {
+        reject(
+          new AppError('COMMAND_FAILED', `${cmd} exited with code ${exitCode}`, {
+            cmd,
+            args,
+            stdout,
+            stderr,
+            exitCode,
+          }),
+        );
+        return;
+      }
+      resolve({ stdout, stderr, exitCode, stdoutBuffer });
+    });
+  });
+}
+
+export async function whichCmd(cmd: string): Promise<boolean> {
+  try {
+    const { shell, args } = resolveWhichArgs(cmd);
+    const result = await runCmd(shell, args, { allowFailure: true });
+    return result.exitCode === 0 && result.stdout.trim().length > 0;
+  } catch {
+    return false;
+  }
+}
+
+export function runCmdSync(cmd: string, args: string[], options: ExecOptions = {}): ExecResult {
+  const result = spawnSync(cmd, args, {
+    cwd: options.cwd,
+    env: options.env,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    encoding: options.binaryStdout ? undefined : 'utf8',
+  });
+
+  if (result.error) {
+    const code = (result.error as NodeJS.ErrnoException).code;
+    if (code === 'ENOENT') {
+      throw new AppError('TOOL_MISSING', `${cmd} not found in PATH`, { cmd }, result.error);
+    }
+    throw new AppError('COMMAND_FAILED', `Failed to run ${cmd}`, { cmd, args }, result.error);
+  }
+
+  const stdoutBuffer = options.binaryStdout
+    ? Buffer.isBuffer(result.stdout)
+      ? result.stdout
+      : Buffer.from(result.stdout ?? '')
+    : undefined;
+  const stdout = options.binaryStdout
+    ? ''
+    : typeof result.stdout === 'string'
+      ? result.stdout
+      : (result.stdout ?? '').toString();
+  const stderr =
+    typeof result.stderr === 'string' ? result.stderr : (result.stderr ?? '').toString();
+  const exitCode = result.status ?? 1;
+
+  if (exitCode !== 0 && !options.allowFailure) {
+    throw new AppError('COMMAND_FAILED', `${cmd} exited with code ${exitCode}`, {
+      cmd,
+      args,
+      stdout,
+      stderr,
+      exitCode,
+    });
+  }
+
+  return { stdout, stderr, exitCode, stdoutBuffer };
+}
+
+export function runCmdDetached(cmd: string, args: string[], options: ExecOptions = {}): void {
+  const child = spawn(cmd, args, {
+    cwd: options.cwd,
+    env: options.env,
+    stdio: 'ignore',
+    detached: true,
+  });
+  child.unref();
+}
+
+export async function runCmdStreaming(
+  cmd: string,
+  args: string[],
+  options: ExecStreamOptions = {},
+): Promise<ExecResult> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, {
+      cwd: options.cwd,
+      env: options.env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+
+    let stdout = '';
+    let stderr = '';
+    let stdoutBuffer: Buffer | undefined = options.binaryStdout ? Buffer.alloc(0) : undefined;
+
+    if (!options.binaryStdout) child.stdout.setEncoding('utf8');
+    child.stderr.setEncoding('utf8');
+
+    child.stdout.on('data', (chunk) => {
+      if (options.binaryStdout) {
+        stdoutBuffer = Buffer.concat([
+          stdoutBuffer ?? Buffer.alloc(0),
+          Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk),
+        ]);
+        return;
+      }
+      const text = String(chunk);
+      stdout += text;
+      options.onStdoutChunk?.(text);
+    });
+
+    child.stderr.on('data', (chunk) => {
+      const text = String(chunk);
+      stderr += text;
+      options.onStderrChunk?.(text);
+    });
+
+    child.on('error', (err) => {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === 'ENOENT') {
+        reject(new AppError('TOOL_MISSING', `${cmd} not found in PATH`, { cmd }, err));
+        return;
+      }
+      reject(new AppError('COMMAND_FAILED', `Failed to run ${cmd}`, { cmd, args }, err));
+    });
+
+    child.on('close', (code) => {
+      const exitCode = code ?? 1;
+      if (exitCode !== 0 && !options.allowFailure) {
+        reject(
+          new AppError('COMMAND_FAILED', `${cmd} exited with code ${exitCode}`, {
+            cmd,
+            args,
+            stdout,
+            stderr,
+            exitCode,
+          }),
+        );
+        return;
+      }
+      resolve({ stdout, stderr, exitCode, stdoutBuffer });
+    });
+  });
+}
+
+export function whichCmdSync(cmd: string): boolean {
+  try {
+    const { shell, args } = resolveWhichArgs(cmd);
+    const result = runCmdSync(shell, args, { allowFailure: true });
+    return result.exitCode === 0 && result.stdout.trim().length > 0;
+  } catch {
+    return false;
+  }
+}
+
+function resolveWhichArgs(cmd: string): { shell: string; args: string[] } {
+  if (process.platform === 'win32') {
+    return { shell: 'cmd.exe', args: ['/c', 'where', cmd] };
+  }
+  return { shell: 'bash', args: ['-lc', `command -v ${cmd}`] };
+}

package/src/utils/interactive.ts

@@ -0,0 +1,4 @@
+export function isInteractive(): boolean {
+  if (process.env.CI) return false;
+  return Boolean(process.stdin.isTTY && process.stdout.isTTY);
+}

package/src/utils/interactors.ts

@@ -0,0 +1,72 @@
+import { AppError } from './errors.ts';
+import type { DeviceInfo } from './device.ts';
+import {
+  closeAndroidApp,
+  fillAndroid,
+  focusAndroid,
+  longPressAndroid,
+  openAndroidApp,
+  pressAndroid,
+  scrollAndroid,
+  scrollIntoViewAndroid,
+  screenshotAndroid,
+  typeAndroid,
+} from '../platforms/android/index.ts';
+import {
+  closeIosApp,
+  fillIos,
+  focusIos,
+  longPressIos,
+  openIosApp,
+  pressIos,
+  scrollIos,
+  scrollIntoViewIos,
+  screenshotIos,
+  typeIos,
+} from '../platforms/ios/index.ts';
+
+export type Interactor = {
+  open(app: string): Promise<void>;
+  close(app: string): Promise<void>;
+  tap(x: number, y: number): Promise<void>;
+  longPress(x: number, y: number, durationMs?: number): Promise<void>;
+  focus(x: number, y: number): Promise<void>;
+  type(text: string): Promise<void>;
+  fill(x: number, y: number, text: string): Promise<void>;
+  scroll(direction: string, amount?: number): Promise<void>;
+  scrollIntoView(text: string): Promise<void>;
+  screenshot(outPath: string): Promise<void>;
+};
+
+export function getInteractor(device: DeviceInfo): Interactor {
+  switch (device.platform) {
+    case 'android':
+      return {
+        open: (app) => openAndroidApp(device, app),
+        close: (app) => closeAndroidApp(device, app),
+        tap: (x, y) => pressAndroid(device, x, y),
+        longPress: (x, y, durationMs) => longPressAndroid(device, x, y, durationMs),
+        focus: (x, y) => focusAndroid(device, x, y),
+        type: (text) => typeAndroid(device, text),
+        fill: (x, y, text) => fillAndroid(device, x, y, text),
+        scroll: (direction, amount) => scrollAndroid(device, direction, amount),
+        scrollIntoView: (text) => scrollIntoViewAndroid(device, text),
+        screenshot: (outPath) => screenshotAndroid(device, outPath),
+      };
+    case 'ios':
+      return {
+        open: (app) => openIosApp(device, app),
+        close: (app) => closeIosApp(device, app),
+        tap: (x, y) => pressIos(device, x, y),
+        longPress: (x, y, durationMs) => longPressIos(device, x, y, durationMs),
+        focus: (x, y) => focusIos(device, x, y),
+        type: (text) => typeIos(device, text),
+        fill: (x, y, text) => fillIos(device, x, y, text),
+        scroll: (direction, amount) => scrollIos(device, direction, amount),
+        scrollIntoView: (text) => scrollIntoViewIos(text),
+        screenshot: (outPath) => screenshotIos(device, outPath),
+      };
+    default:
+      throw new AppError('UNSUPPORTED_PLATFORM', `Unsupported platform: ${device.platform}`);
+  }
+}

package/src/utils/output.ts

@@ -0,0 +1,146 @@
+import { AppError } from './errors.ts';
+
+export type JsonResult =
+  | { success: true; data?: Record<string, unknown> }
+  | { success: false; error: { code: string; message: string; details?: Record<string, unknown> } };
+
+export function printJson(result: JsonResult): void {
+  process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+}
+
+export function printHumanError(err: AppError): void {
+  const details = err.details ? `\n${JSON.stringify(err.details, null, 2)}` : '';
+  process.stderr.write(`Error (${err.code}): ${err.message}${details}\n`);
+}
+
+type SnapshotRect = { x: number; y: number; width: number; height: number };
+type SnapshotNode = {
+  ref?: string;
+  depth?: number;
+  type?: string;
+  label?: string;
+  value?: string;
+  identifier?: string;
+  rect?: SnapshotRect;
+  hittable?: boolean;
+  enabled?: boolean;
+};
+
+export function formatSnapshotText(
+  data: Record<string, unknown>,
+  options: { raw?: boolean } = {},
+): string {
+  const nodes = (data.nodes ?? []) as SnapshotNode[];
+  const truncated = Boolean(data.truncated);
+  const appName = typeof data.appName === 'string' ? data.appName : undefined;
+  const appBundleId = typeof data.appBundleId === 'string' ? data.appBundleId : undefined;
+  const meta: string[] = [];
+  if (appName) meta.push(`Page: ${appName}`);
+  if (appBundleId) meta.push(`App: ${appBundleId}`);
+  const header = `Snapshot: ${nodes.length} nodes${truncated ? ' (truncated)' : ''}`;
+  const prefix = meta.length > 0 ? `${meta.join('\n')}\n` : '';
+  if (!Array.isArray(nodes) || nodes.length === 0) {
+    return `${prefix}${header}\n`;
+  }
+  if (options.raw) {
+    const rawLines = nodes.map((node) => JSON.stringify(node));
+    return `${prefix}${header}\n${rawLines.join('\n')}\n`;
+  }
+  const hiddenGroupDepths: number[] = [];
+  const lines: string[] = [];
+  for (const node of nodes) {
+    const depth = node.depth ?? 0;
+    while (hiddenGroupDepths.length > 0 && depth <= hiddenGroupDepths[hiddenGroupDepths.length - 1]) {
+      hiddenGroupDepths.pop();
+    }
+    const label = node.label?.trim() || node.value?.trim() || node.identifier?.trim() || '';
+    const type = formatRole(node.type ?? 'Element');
+    const isHiddenGroup = type === 'group' && !label;
+    if (isHiddenGroup) {
+      hiddenGroupDepths.push(depth);
+    }
+    const adjustedDepth = isHiddenGroup
+      ? depth
+      : Math.max(0, depth - hiddenGroupDepths.length);
+    const indent = '  '.repeat(adjustedDepth);
+    const ref = node.ref ? `@${node.ref}` : '';
+    const flags = [
+      node.enabled === false ? 'disabled' : null,
+    ]
+      .filter(Boolean)
+      .join(', ');
+    const flagText = flags ? ` [${flags}]` : '';
+    const textPart = label ? ` "${label}"` : '';
+    if (isHiddenGroup) {
+      lines.push(`${indent}${ref} [${type}]${flagText}`.trimEnd());
+      continue;
+    }
+    lines.push(`${indent}${ref} [${type}]${textPart}${flagText}`.trimEnd());
+  }
+  return `${prefix}${header}\n${lines.join('\n')}\n`;
+}
+
+function formatRole(type: string): string {
+  let normalized = type.replace(/XCUIElementType/gi, '').toLowerCase();
+  if (normalized.startsWith("ax")) {
+    normalized = normalized.replace(/^ax/, "");
+  }
+  switch (normalized) {
+    case 'application':
+      return 'application';
+    case 'navigationbar':
+      return 'navigation-bar';
+    case 'tabbar':
+      return 'tab-bar';
+    case 'button':
+      return 'button';
+    case 'link':
+      return 'link';
+    case 'cell':
+      return 'cell';
+    case 'statictext':
+      return 'text';
+    case 'textfield':
+      return 'text-field';
+    case 'textview':
+      return 'text-view';
+    case 'switch':
+      return 'switch';
+    case 'slider':
+      return 'slider';
+    case 'image':
+      return 'image';
+    case 'table':
+      return 'list';
+    case 'collectionview':
+      return 'collection';
+    case 'searchfield':
+      return 'search';
+    case 'segmentedcontrol':
+      return 'segmented-control';
+    case 'group':
+      return 'group';
+    case 'window':
+      return 'window';
+    case 'statictext':
+      return 'text';
+    case 'textfield':
+      return 'text-field';
+    case 'textarea':
+      return 'text-view';
+    case 'checkbox':
+      return 'checkbox';
+    case 'radio':
+      return 'radio';
+    case 'menuitem':
+      return 'menu-item';
+    case 'toolbar':
+      return 'toolbar';
+    case 'scrollarea':
+      return 'scroll-area';
+    case 'table':
+      return 'table';
+    default:
+      return normalized || 'element';
+  }
+}

package/src/utils/snapshot.ts

@@ -0,0 +1,63 @@
+export type Rect = {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+};
+
+export type SnapshotOptions = {
+  interactiveOnly?: boolean;
+  compact?: boolean;
+  depth?: number;
+  scope?: string;
+  raw?: boolean;
+};
+
+export type RawSnapshotNode = {
+  index: number;
+  type?: string;
+  label?: string;
+  value?: string;
+  identifier?: string;
+  rect?: Rect;
+  enabled?: boolean;
+  hittable?: boolean;
+  depth?: number;
+  parentIndex?: number;
+};
+
+export type SnapshotNode = RawSnapshotNode & {
+  ref: string;
+};
+
+export type SnapshotState = {
+  nodes: SnapshotNode[];
+  createdAt: number;
+  truncated?: boolean;
+  backend?: 'ax' | 'xctest' | 'android';
+};
+
+export function attachRefs(nodes: RawSnapshotNode[]): SnapshotNode[] {
+  return nodes.map((node, idx) => ({ ...node, ref: `e${idx + 1}` }));
+}
+
+export function normalizeRef(input: string): string | null {
+  const trimmed = input.trim();
+  if (trimmed.startsWith('@')) {
+    const ref = trimmed.slice(1);
+    return ref ? ref : null;
+  }
+  if (trimmed.startsWith('e')) return trimmed;
+  return null;
+}
+
+export function findNodeByRef(nodes: SnapshotNode[], ref: string): SnapshotNode | null {
+  return nodes.find((node) => node.ref === ref) ?? null;
+}
+
+export function centerOfRect(rect: Rect): { x: number; y: number } {
+  return {
+    x: Math.round(rect.x + rect.width / 2),
+    y: Math.round(rect.y + rect.height / 2),
+  };
+}