neoagent 2.1.12 → 2.1.14

package/server/services/android/controller.js

@@ -276,6 +276,49 @@ function extractZip(zipPath, destDir) {
   throw new Error('Neither unzip nor ditto is available to extract Android SDK archives');
 }
 
+function listFilesRecursive(rootDir, predicate, bucket = []) {
+  for (const entry of fs.readdirSync(rootDir, { withFileTypes: true })) {
+    const fullPath = path.join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      listFilesRecursive(fullPath, predicate, bucket);
+      continue;
+    }
+    if (!predicate || predicate(fullPath, entry)) {
+      bucket.push(fullPath);
+    }
+  }
+  return bucket;
+}
+
+function resolveBundleInstallTargets(bundleDir) {
+  const apkFiles = listFilesRecursive(bundleDir, (filePath) => path.extname(filePath).toLowerCase() === '.apk')
+    .sort((a, b) => a.localeCompare(b));
+  if (apkFiles.length === 0) {
+    throw new Error('APK bundle did not contain any installable .apk files.');
+  }
+
+  const universalApk = apkFiles.find((filePath) => path.basename(filePath).toLowerCase() === 'universal.apk');
+  if (universalApk) {
+    return {
+      mode: 'single',
+      installPaths: [universalApk],
+      layout: 'universal',
+    };
+  }
+
+  if (apkFiles.length === 1) {
+    return {
+      mode: 'single',
+      installPaths: apkFiles,
+      layout: 'single-apk',
+    };
+  }
+
+  throw new Error(
+    'APK bundles must include a universal APK. Export a universal .apks bundle or upload a single .apk instead.'
+  );
+}
+
 function parseLatestCmdlineToolsUrl(xml) {
   const tag = platformTag() === 'mac' ? 'macosx' : 'linux';
   const packageMatch = xml.match(new RegExp(`<remotePackage\\s+path="cmdline-tools;latest">([\\s\\S]*?)<\\/remotePackage>`));
@@ -1104,7 +1147,7 @@ class AndroidController {
   }
 
   async screenshot(options = {}) {
-    const serial = await this.ensureDevice();
+    const serial = options.serial || await this.ensureDevice();
     const filename = `android_${Date.now()}.png`;
     const fullPath = path.join(SCREENSHOTS_DIR, filename);
     await this.#run(`${quoteShell(adbBinary())} -s ${quoteShell(serial)} exec-out screencap -p > ${quoteShell(fullPath)}`, { timeout: 30000 });
@@ -1117,7 +1160,7 @@ class AndroidController {
   }
 
   async dumpUi(options = {}) {
-    const serial = await this.ensureDevice();
+    const serial = options.serial || await this.ensureDevice();
     let xml = await this.#adb(serial, 'shell uiautomator dump --compressed /dev/tty', { timeout: 30000 });
     if (!String(xml || '').includes('<hierarchy')) {
       const remote = '/sdcard/neoagent-ui.xml';
@@ -1140,6 +1183,65 @@ class AndroidController {
     };
   }
 
+  async #captureObservation(serial, options = {}) {
+    const resolvedSerial = serial || await this.ensureDevice();
+    const observation = {
+      serial: resolvedSerial,
+      screenshotPath: null,
+      fullPath: null,
+      uiDumpPath: null,
+      nodeCount: null,
+      preview: undefined,
+      observationWarnings: [],
+    };
+
+    if (options.screenshot !== false) {
+      try {
+        const shot = await this.screenshot({ serial: resolvedSerial });
+        observation.screenshotPath = shot?.screenshotPath || null;
+        observation.fullPath = shot?.fullPath || null;
+      } catch (err) {
+        observation.observationWarnings.push(`screenshot: ${err.message}`);
+      }
+    }
+
+    if (options.uiDump !== false) {
+      try {
+        const dump = await this.dumpUi({
+          serial: resolvedSerial,
+          includeNodes: options.includeNodes !== false,
+        });
+        observation.uiDumpPath = dump.uiDumpPath;
+        observation.nodeCount = dump.nodeCount;
+        observation.preview = dump.preview;
+      } catch (err) {
+        observation.observationWarnings.push(`ui_dump: ${err.message}`);
+      }
+    }
+
+    if (observation.observationWarnings.length === 0) {
+      delete observation.observationWarnings;
+    }
+
+    return observation;
+  }
+
+  async observe(options = {}) {
+    const serial = options.serial || await this.ensureDevice();
+    const observation = await this.#captureObservation(serial, options);
+    if (!observation.screenshotPath && !observation.uiDumpPath) {
+      throw new Error(
+        Array.isArray(observation.observationWarnings) && observation.observationWarnings.length > 0
+          ? observation.observationWarnings.join(' | ')
+          : 'Unable to capture Android observation',
+      );
+    }
+    return {
+      success: true,
+      ...observation,
+    };
+  }
+
   async #resolveSelector(args = {}) {
     const dump = await this.dumpUi({ includeNodes: false });
     const selector = {
@@ -1164,27 +1266,27 @@ class AndroidController {
     let y = Number(args.y);
     let node = null;
     let serial = await this.ensureDevice();
-    let uiDumpPath = null;
+    let resolvedFromUiDumpPath = null;
 
     if (!Number.isFinite(x) || !Number.isFinite(y)) {
       const resolved = await this.#resolveSelector(args);
       serial = resolved.serial;
       node = resolved.node;
-      uiDumpPath = resolved.uiDumpPath;
+      resolvedFromUiDumpPath = resolved.uiDumpPath;
       x = node.bounds.centerX;
       y = node.bounds.centerY;
     }
 
     await this.#adb(serial, `shell input tap ${Math.round(x)} ${Math.round(y)}`, { timeout: 15000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial, args);
     return {
       success: true,
       serial,
       x: Math.round(x),
       y: Math.round(y),
       target: summarizeNode(node),
-      uiDumpPath,
-      screenshotPath: shot.screenshotPath,
+      resolvedFromUiDumpPath,
+      ...observation,
     };
   }
 
@@ -1193,13 +1295,13 @@ class AndroidController {
     let y = Number(args.y);
     let node = null;
     let serial = await this.ensureDevice();
-    let uiDumpPath = null;
+    let resolvedFromUiDumpPath = null;
 
     if (!Number.isFinite(x) || !Number.isFinite(y)) {
       const resolved = await this.#resolveSelector(args);
       serial = resolved.serial;
       node = resolved.node;
-      uiDumpPath = resolved.uiDumpPath;
+      resolvedFromUiDumpPath = resolved.uiDumpPath;
       x = node.bounds.centerX;
       y = node.bounds.centerY;
     }
@@ -1210,7 +1312,7 @@ class AndroidController {
       `shell input swipe ${Math.round(x)} ${Math.round(y)} ${Math.round(x)} ${Math.round(y)} ${Math.round(durationMs)}`,
       { timeout: Math.max(15000, durationMs + 5000) },
     );
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial, args);
     return {
       success: true,
       serial,
@@ -1218,8 +1320,8 @@ class AndroidController {
       y: Math.round(y),
       durationMs,
       target: summarizeNode(node),
-      uiDumpPath,
-      screenshotPath: shot.screenshotPath,
+      resolvedFromUiDumpPath,
+      ...observation,
     };
   }
 
@@ -1237,6 +1339,8 @@ class AndroidController {
         description: args.description,
         className: args.className,
         clickable: true,
+        screenshot: false,
+        uiDump: false,
       }).catch(() => {});
     }
 
@@ -1244,12 +1348,12 @@ class AndroidController {
     if (args.pressEnter) {
       await this.#adb(serial, 'shell input keyevent 66', { timeout: 10000 });
     }
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial, args);
     return {
       success: true,
       serial,
       typed: args.text || '',
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1264,11 +1368,11 @@ class AndroidController {
       throw new Error('x1, y1, x2, and y2 are required for android_swipe');
     }
     await this.#adb(serial, `shell input swipe ${Math.round(x1)} ${Math.round(y1)} ${Math.round(x2)} ${Math.round(y2)} ${Math.round(duration)}`, { timeout: 15000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial, args);
     return {
       success: true,
       serial,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1278,13 +1382,13 @@ class AndroidController {
     const keyCode = Number.isFinite(Number(raw)) ? Number(raw) : (DEFAULT_KEYEVENTS[raw] || null);
     if (!keyCode) throw new Error(`Unsupported Android key: ${args.key}`);
     await this.#adb(serial, `shell input keyevent ${keyCode}`, { timeout: 10000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial, args);
     return {
       success: true,
       serial,
       key: args.key,
       keyCode,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1304,13 +1408,17 @@ class AndroidController {
         clickable: args.clickable,
       });
       if (node) {
-        const shot = args.screenshot === false ? null : await this.screenshot();
+        const observation = await this.#captureObservation(dump.serial, {
+          screenshot: args.screenshot !== false,
+          uiDump: args.uiDump !== false,
+          includeNodes: args.includeNodes,
+        });
         return {
           success: true,
           serial: dump.serial,
           matched: summarizeNode(node),
-          uiDumpPath: dump.uiDumpPath,
-          screenshotPath: shot?.screenshotPath || null,
+          matchedFromUiDumpPath: dump.uiDumpPath,
+          ...observation,
         };
       }
       await sleep(intervalMs);
@@ -1341,13 +1449,13 @@ class AndroidController {
     } else {
       throw new Error('packageName is required for android_open_app');
     }
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial, args);
     return {
       success: true,
       serial,
       packageName: args.packageName,
       activity: args.activity || null,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1367,11 +1475,11 @@ class AndroidController {
     }
 
     await this.#adb(serial, parts.join(' '), { timeout: 20000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial, args);
     return {
       success: true,
       serial,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1397,11 +1505,38 @@ class AndroidController {
     const apkPath = path.resolve(String(args.apkPath || ''));
     if (!apkPath || !fs.existsSync(apkPath)) throw new Error(`APK not found: ${apkPath}`);
     const serial = await this.ensureDevice();
+    const extension = path.extname(apkPath).toLowerCase();
+
+    if (extension === '.aab') {
+      throw new Error('.aab app bundles are not directly installable. Export a .apks bundle or .apk first.');
+    }
+
+    if (extension === '.apks') {
+      const extractDir = fs.mkdtempSync(path.join(TMP_DIR, 'apk-bundle-'));
+      try {
+        extractZip(apkPath, extractDir);
+        const bundle = resolveBundleInstallTargets(extractDir);
+        await this.#adb(serial, `install -r ${quoteShell(bundle.installPaths[0])}`, { timeout: 300000 });
+        return {
+          success: true,
+          serial,
+          apkPath,
+          artifactType: 'apks',
+          installedPaths: bundle.installPaths,
+          bundleLayout: bundle.layout,
+        };
+      } finally {
+        fs.rmSync(extractDir, { recursive: true, force: true });
+      }
+    }
+
     await this.#adb(serial, `install -r ${quoteShell(apkPath)}`, { timeout: 300000 });
     return {
       success: true,
       serial,
       apkPath,
+      artifactType: 'apk',
+      installedPaths: [apkPath],
     };
   }
 
@@ -1412,13 +1547,19 @@ class AndroidController {
 
     const timeout = Math.max(1000, Number(args.timeoutMs) || 20000);
     const stdout = await this.#adb(serial, `shell ${quoteShell(command)}`, { timeout });
-    const shot = args.screenshot === true ? await this.screenshot() : null;
+    const observation = args.screenshot === true
+      ? await this.#captureObservation(serial)
+      : null;
     return {
       success: true,
       serial,
       command,
       stdout,
-      screenshotPath: shot?.screenshotPath || null,
+      screenshotPath: observation?.screenshotPath || null,
+      fullPath: observation?.fullPath || null,
+      uiDumpPath: observation?.uiDumpPath || null,
+      nodeCount: observation?.nodeCount,
+      preview: observation?.preview,
     };
   }
 

package/server/services/websocket.js

@@ -76,6 +76,25 @@ function setupWebSocket(io, services) {
           }
         }
 
+        const activeRun = agentEngine.findSteerableRunForUser(userId, 'web');
+        if (activeRun) {
+          const queued = agentEngine.enqueueSteering(activeRun.runId, task, {
+            platform: 'web',
+            socketId: socket.id
+          });
+          if (queued) {
+            db.prepare('INSERT INTO conversation_history (user_id, agent_run_id, role, content, metadata) VALUES (?, ?, ?, ?, ?)')
+              .run(
+                userId,
+                activeRun.runId,
+                'user',
+                task,
+                JSON.stringify({ platform: 'web', steering: true })
+              );
+            return;
+          }
+        }
+
         db.prepare('INSERT INTO conversation_history (user_id, role, content, metadata) VALUES (?, ?, ?, ?)')
           .run(userId, 'user', task, JSON.stringify({ platform: 'web' }));