neoagent 2.1.10 → 2.1.12

package/server/services/ai/tools.js

@@ -213,6 +213,24 @@ function getAvailableTools(app, options = {}) {
                 }
             }
         },
+        {
+            name: 'android_long_press',
+            description: 'Long-press an Android UI element or screen coordinate. Useful for context menus, drag handles, rearranging icons, and long-click actions.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    x: { type: 'number', description: 'Absolute X coordinate' },
+                    y: { type: 'number', description: 'Absolute Y coordinate' },
+                    text: { type: 'string', description: 'Visible text to match in the UI dump' },
+                    resourceId: { type: 'string', description: 'Android resource-id to match' },
+                    description: { type: 'string', description: 'content-desc / accessibility label to match' },
+                    className: { type: 'string', description: 'Optional class name filter' },
+                    packageName: { type: 'string', description: 'Optional package filter' },
+                    clickable: { type: 'boolean', description: 'Prefer clickable elements' },
+                    durationMs: { type: 'number', description: 'Press duration in milliseconds (default 650)' }
+                }
+            }
+        },
         {
             name: 'android_type',
             description: 'Type text into the focused Android field, optionally tapping a matched element first.',
@@ -313,6 +331,19 @@ function getAvailableTools(app, options = {}) {
                 required: ['apkPath']
             }
         },
+        {
+            name: 'android_shell',
+            description: 'Run an adb shell command on the active Android device or emulator. Use this when a needed phone action is not covered by a higher-level Android tool.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    command: { type: 'string', description: 'Shell command to run on-device, without the leading "adb shell"' },
+                    timeoutMs: { type: 'number', description: 'Timeout in milliseconds (default 20000)' },
+                    screenshot: { type: 'boolean', description: 'Capture a screenshot after the command if it changes the UI (default false)' }
+                },
+                required: ['command']
+            }
+        },
         {
             name: 'web_search',
             description: 'Search the public web without opening the browser. Uses Brave Search API for fast result retrieval.',
@@ -879,6 +910,12 @@ async function executeTool(toolName, args, context, engine) {
             return await controller.tap(args || {});
         }
 
+        case 'android_long_press': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.longPress(args || {});
+        }
+
         case 'android_type': {
             const controller = ac();
             if (!controller) return { error: 'Android controller not available' };
@@ -927,6 +964,12 @@ async function executeTool(toolName, args, context, engine) {
             return await controller.installApk(args || {});
         }
 
+        case 'android_shell': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.shell(args || {});
+        }
+
         case 'web_search': {
             const apiKey = process.env.BRAVE_SEARCH_API_KEY;
             if (!apiKey) return { error: 'BRAVE_SEARCH_API_KEY is not configured' };

package/server/services/android/controller.js

@@ -64,6 +64,25 @@ function commandExists(command) {
   return probe.status === 0;
 }
 
+function parseResolvedLaunchComponent(output, packageName) {
+  const lines = String(output || '')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean);
+  const normalizedPackage = String(packageName || '').trim();
+  const componentPattern = /^[A-Za-z0-9._$]+\/[A-Za-z0-9._$]+$/;
+  const relativePattern = /^[A-Za-z0-9._$]+\/\.[A-Za-z0-9._$]+$/;
+
+  const exact = lines.find((line) =>
+    normalizedPackage
+      ? line.startsWith(`${normalizedPackage}/`)
+      : componentPattern.test(line) || relativePattern.test(line)
+  );
+  if (exact) return exact;
+
+  return lines.find((line) => componentPattern.test(line) || relativePattern.test(line)) || null;
+}
+
 function appendState(patch) {
   const current = readState();
   const next = {
@@ -470,6 +489,39 @@ function systemImagePackageToRelativeDir(packageName) {
   return `${parts.join('/')}/`;
 }
 
+function systemImagePackageToAbi(packageName) {
+  const parts = String(packageName || '').split(';').filter(Boolean);
+  if (parts.length !== 4 || parts[0] !== 'system-images') {
+    return null;
+  }
+  return parts[3] || null;
+}
+
+function abiToCpuArch(abi) {
+  const value = String(abi || '').trim().toLowerCase();
+  if (value === 'arm64-v8a') return 'arm64';
+  if (value === 'armeabi-v7a' || value === 'armeabi') return 'arm';
+  if (value === 'x86_64') return 'x86_64';
+  if (value === 'x86') return 'x86';
+  return null;
+}
+
+function systemImagePackageToCpuArch(packageName) {
+  return abiToCpuArch(systemImagePackageToAbi(packageName));
+}
+
+function describeAutoFixChanges(current, next, fields = []) {
+  return fields
+    .map((field) => {
+      const before = current?.[field] ?? null;
+      const after = next?.[field] ?? null;
+      if (before === after) return null;
+      return `${field}: ${before ?? 'null'} -> ${after ?? 'null'}`;
+    })
+    .filter(Boolean)
+    .join(', ');
+}
+
 function sanitizeUiXml(raw) {
   const text = String(raw || '');
   const start = text.indexOf('<?xml');
@@ -581,7 +633,28 @@ class AndroidController {
         if (!preferredInstalled) {
           throw new Error(formatSystemImageError(installedImages));
         }
-        if (preferredInstalled.packageName !== state.systemImage) {
+        const stateApiLevel = Number(state.apiLevel || 0) || 0;
+        const stateNeedsRefresh =
+          preferredInstalled.packageName !== state.systemImage ||
+          preferredInstalled.apiLevel !== stateApiLevel ||
+          preferredInstalled.arch !== state.systemImageArch;
+        if (stateNeedsRefresh) {
+          const changeSummary = describeAutoFixChanges(
+            {
+              systemImage: state.systemImage || null,
+              apiLevel: stateApiLevel || null,
+              systemImageArch: state.systemImageArch || null,
+            },
+            {
+              systemImage: preferredInstalled.packageName,
+              apiLevel: preferredInstalled.apiLevel,
+              systemImageArch: preferredInstalled.arch,
+            },
+            ['systemImage', 'apiLevel', 'systemImageArch']
+          );
+          if (changeSummary) {
+            console.log(`[Android] Auto-fixed preferred system image (${changeSummary})`);
+          }
           appendState({
             bootstrapped: true,
             systemImage: preferredInstalled.packageName,
@@ -670,19 +743,39 @@ class AndroidController {
     if (!pkg) throw new Error('Android system image not installed');
     const avdExists = list.includes(`Name: ${this.avdName}`);
     let avdNeedsRecreate = avdExists && (!state.avdSystemImage || state.avdSystemImage !== pkg);
+    const avdRecreateReasons = [];
+    if (avdNeedsRecreate && state.avdSystemImage !== pkg) {
+      avdRecreateReasons.push(`systemImage: ${state.avdSystemImage || 'null'} -> ${pkg}`);
+    }
     const configPath = path.join(AVD_HOME, `${this.avdName}.avd`, 'config.ini');
     if (avdExists && fs.existsSync(configPath)) {
       try {
         const config = fs.readFileSync(configPath, 'utf8');
         const currentImageDir = readIniValue(config, 'image.sysdir.1');
         const expectedImageDir = systemImagePackageToRelativeDir(pkg);
+        const currentAbi = readIniValue(config, 'abi.type');
+        const expectedAbi = systemImagePackageToAbi(pkg);
+        const currentCpuArch = readIniValue(config, 'hw.cpu.arch');
+        const expectedCpuArch = systemImagePackageToCpuArch(pkg);
         if (expectedImageDir && currentImageDir && currentImageDir !== expectedImageDir) {
           avdNeedsRecreate = true;
+          avdRecreateReasons.push(`image.sysdir.1: ${currentImageDir} -> ${expectedImageDir}`);
+        }
+        if (expectedAbi && currentAbi && currentAbi !== expectedAbi) {
+          avdNeedsRecreate = true;
+          avdRecreateReasons.push(`abi.type: ${currentAbi} -> ${expectedAbi}`);
+        }
+        if (expectedCpuArch && currentCpuArch && currentCpuArch !== expectedCpuArch) {
+          avdNeedsRecreate = true;
+          avdRecreateReasons.push(`hw.cpu.arch: ${currentCpuArch} -> ${expectedCpuArch}`);
         }
       } catch {}
     }
 
     if (avdNeedsRecreate) {
+      if (avdRecreateReasons.length > 0) {
+        console.log(`[Android] Recreating AVD to repair config mismatch (${avdRecreateReasons.join(', ')})`);
+      }
       await this.stopEmulator().catch(() => {});
       await this.#run(`${quoteShell(avdManagerBinary())} delete avd -n ${quoteShell(this.avdName)}`, {
         timeout: 120000,
@@ -1095,6 +1188,41 @@ class AndroidController {
     };
   }
 
+  async longPress(args = {}) {
+    let x = Number(args.x);
+    let y = Number(args.y);
+    let node = null;
+    let serial = await this.ensureDevice();
+    let uiDumpPath = null;
+
+    if (!Number.isFinite(x) || !Number.isFinite(y)) {
+      const resolved = await this.#resolveSelector(args);
+      serial = resolved.serial;
+      node = resolved.node;
+      uiDumpPath = resolved.uiDumpPath;
+      x = node.bounds.centerX;
+      y = node.bounds.centerY;
+    }
+
+    const durationMs = Math.max(250, Number(args.durationMs) || 650);
+    await this.#adb(
+      serial,
+      `shell input swipe ${Math.round(x)} ${Math.round(y)} ${Math.round(x)} ${Math.round(y)} ${Math.round(durationMs)}`,
+      { timeout: Math.max(15000, durationMs + 5000) },
+    );
+    const shot = await this.screenshot();
+    return {
+      success: true,
+      serial,
+      x: Math.round(x),
+      y: Math.round(y),
+      durationMs,
+      target: summarizeNode(node),
+      uiDumpPath,
+      screenshotPath: shot.screenshotPath,
+    };
+  }
+
   async type(args = {}) {
     const serial = await this.ensureDevice();
     if (args.clear === true) {
@@ -1196,7 +1324,20 @@ class AndroidController {
     if (args.activity) {
       await this.#adb(serial, `shell am start -n ${quoteShell(`${args.packageName}/${args.activity}`)}`, { timeout: 20000 });
     } else if (args.packageName) {
-      await this.#adb(serial, `shell monkey -p ${quoteShell(args.packageName)} -c android.intent.category.LAUNCHER 1`, { timeout: 30000 });
+      const resolved = await this.#runAllowFailure(
+        `${quoteShell(adbBinary())} -s ${quoteShell(serial)} shell cmd package resolve-activity --brief -c android.intent.category.LAUNCHER ${quoteShell(args.packageName)}`,
+        { timeout: 15000 },
+      );
+      const component = parseResolvedLaunchComponent(
+        `${resolved.stdout || ''}\n${resolved.stderr || ''}`,
+        args.packageName,
+      );
+
+      if (component) {
+        await this.#adb(serial, `shell am start -n ${quoteShell(component)}`, { timeout: 20000 });
+      } else {
+        await this.#adb(serial, `shell monkey -p ${quoteShell(args.packageName)} -c android.intent.category.LAUNCHER 1`, { timeout: 30000 });
+      }
     } else {
       throw new Error('packageName is required for android_open_app');
     }
@@ -1264,6 +1405,23 @@ class AndroidController {
     };
   }
 
+  async shell(args = {}) {
+    const serial = await this.ensureDevice();
+    const command = String(args.command || '').trim();
+    if (!command) throw new Error('command is required for android_shell');
+
+    const timeout = Math.max(1000, Number(args.timeoutMs) || 20000);
+    const stdout = await this.#adb(serial, `shell ${quoteShell(command)}`, { timeout });
+    const shot = args.screenshot === true ? await this.screenshot() : null;
+    return {
+      success: true,
+      serial,
+      command,
+      stdout,
+      screenshotPath: shot?.screenshotPath || null,
+    };
+  }
+
   async getStatus() {
     const devices = isExecutable(adbBinary())
       ? await this.listDevices({ ensureBootstrapped: false }).catch(() => [])
@@ -1332,6 +1490,7 @@ module.exports = {
   configuredSystemImagePackage,
   configuredSystemImagePlatform,
   formatSystemImageError,
+  parseResolvedLaunchComponent,
   parseLatestCmdlineToolsUrl,
   parseSystemImages,
   sanitizeUiXml,