neoagent 2.4.0 → 2.4.1-beta.6

package/server/services/ai/tools.js

@@ -292,7 +292,7 @@ function getAvailableTools(app, options = {}) {
     const tools = [
         {
             name: 'execute_command',
-            description: 'Execute a terminal/shell command as a normal recoverable agent step. Waits for the process to exit, supports PTY for interactive programs, and returns stdout, stderr, exit code, timeout state, and duration so later reasoning can inspect failures, install missing dependencies, and retry when needed.',
+            description: 'Execute a terminal/shell command as a normal recoverable agent step. Waits for the process to exit, supports PTY for interactive programs, and returns stdout, stderr, exit code, timeout state, duration, and a backend field ("vm" or "desktop-companion") indicating where the command ran. Commands run inside the isolated VM unless the cli_backend setting is set to "desktop" and a companion app is connected, in which case the command runs on the companion desktop machine.',
             parameters: {
                 type: 'object',
                 properties: {
@@ -308,7 +308,7 @@ function getAvailableTools(app, options = {}) {
         },
         {
             name: 'browser_navigate',
-            description: 'Navigate the browser to a URL and return page content/screenshot',
+            description: 'Navigate the browser to a URL and return page content/screenshot. The result includes a backend field ("vm" or "extension") indicating whether the VM browser or the paired browser extension handled the request.',
             parameters: {
                 type: 'object',
                 properties: {
@@ -1401,10 +1401,13 @@ async function executeTool(toolName, args, context, engine) {
         allowMultipleProactiveMessages = false
     } = context;
     const runtime = () => app?.locals?.runtimeManager || engine.runtimeManager || null;
-    const bc = () => {
+    const bc = async () => {
         const manager = runtime();
         if (manager && typeof manager.getBrowserProviderForUser === 'function') {
-            return manager.getBrowserProviderForUser(userId);
+            const backend = typeof manager.getActiveBrowserBackend === 'function'
+                ? manager.getActiveBrowserBackend(userId)
+                : 'vm';
+            return { provider: await manager.getBrowserProviderForUser(userId), backend };
         }
         throw new Error('Browser provider is unavailable. VM runtime is required.');
     };
@@ -1478,59 +1481,67 @@ async function executeTool(toolName, args, context, engine) {
 
         case 'execute_command': {
             const runtimeManager = runtime();
-            if (!runtimeManager || typeof runtimeManager.executeCommand !== 'function') {
-                return { error: 'Command execution is unavailable. VM runtime is required.' };
+            if (!runtimeManager) {
+                return { error: 'Command execution is unavailable. No runtime manager found.' };
             }
-            return await runtimeManager.executeCommand(userId, args.command, {
+            const execOptions = {
                 cwd: args.cwd,
                 timeout: args.timeout || (args.pty ? 20 * 60 * 1000 : 15 * 60 * 1000),
                 stdinInput: args.stdin_input,
                 pty: args.pty === true,
                 inputs: args.inputs || [],
-            });
+            };
+            if (typeof runtimeManager.executeCliCommand === 'function') {
+                return await runtimeManager.executeCliCommand(userId, args.command, execOptions);
+            }
+            // Legacy fallback — older runtime manager without CLI routing.
+            if (typeof runtimeManager.executeCommand !== 'function') {
+                return { error: 'Command execution is unavailable. VM runtime is required.' };
+            }
+            return { ...await runtimeManager.executeCommand(userId, args.command, execOptions), backend: 'vm' };
         }
 
         case 'browser_navigate': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.navigate(args.url, {
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.navigate(args.url, {
                 screenshot: args.screenshot !== false,
                 waitFor: args.waitFor,
                 fullPage: args.fullPage
-            });
+            }), backend };
         }
 
         case 'browser_click': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.click(args.selector, args.text, args.screenshot !== false);
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.click(args.selector, args.text, args.screenshot !== false), backend };
         }
 
         case 'browser_type': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.type(args.selector, args.text, {
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.type(args.selector, args.text, {
                 clear: args.clear !== false,
                 pressEnter: args.pressEnter
-            });
+            }), backend };
         }
 
         case 'browser_extract': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.extract(args.selector, args.attribute, args.all);
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.extract(args.selector, args.attribute, args.all), backend };
         }
 
         case 'browser_screenshot': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.screenshot({ fullPage: args.fullPage, selector: args.selector });
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.screenshot({ fullPage: args.fullPage, selector: args.selector }), backend };
         }
 
         case 'browser_evaluate': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.evaluate(args.script);
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.evaluate(args.script), backend };
         }
 
         case 'android_start_emulator': {

package/server/services/desktop/protocol.js

@@ -14,6 +14,7 @@ const DESKTOP_COMMANDS = Object.freeze({
   SELECT_DISPLAY: 'selectDisplay',
   GET_TREE: 'getTree',
   PAUSE_CONTROL: 'pauseControl',
+  EXECUTE_COMMAND: 'executeCommand',
   PING: 'ping',
 });
 

package/server/services/desktop/provider.js

@@ -169,6 +169,17 @@ class DesktopProvider {
   getAccessibilityTree(options = {}) {
     return this._dispatch(DESKTOP_COMMANDS.GET_TREE, options);
   }
+
+  executeCommand(command, options = {}) {
+    return this._dispatch(DESKTOP_COMMANDS.EXECUTE_COMMAND, {
+      command,
+      cwd: options.cwd || null,
+      timeout: options.timeout || null,
+      stdin_input: options.stdinInput || null,
+      pty: options.pty === true,
+      inputs: options.inputs || [],
+    });
+  }
 }
 
 module.exports = {

package/server/services/desktop/registry.js

@@ -127,6 +127,16 @@ class DesktopCompanionRegistry {
       now,
     );
 
+    // Remove stale offline entries for the same machine (e.g. after a re-install
+    // that generated a new device_id but kept the same hostname).
+    const hostname = hello.hostname ? String(hello.hostname).trim() : null;
+    if (hostname) {
+      this.db.prepare(
+        `DELETE FROM desktop_companion_devices
+         WHERE user_id = ? AND hostname = ? AND device_id != ? AND status = 'offline'`
+      ).run(userId, hostname, hello.deviceId);
+    }
+
     return this.getDeviceRecordByDeviceId(userId, hello.deviceId);
   }
 
@@ -172,9 +182,6 @@ class DesktopCompanionRegistry {
     const record = this._upsertDeviceRecord(userId, hello, sessionId);
     const userMap = this._getUserMap(userId, true);
     const existing = userMap.get(record.deviceId);
-    if (existing && existing.ws !== ws) {
-      existing.close('replaced by a newer desktop companion connection');
-    }
 
     const connection = new DesktopCompanionConnection({
       registry: this,
@@ -192,7 +199,16 @@ class DesktopCompanionRegistry {
       },
       timeoutMs: this.commandTimeoutMs,
     });
+    // Install the new connection in the map BEFORE closing the old one.
+    // This ensures that when the old socket's async 'close' event fires and
+    // calls unregisterConnection, it sees the new connection as the owner and
+    // skips the DB status='offline' write — preventing a false offline report.
     userMap.set(record.deviceId, connection);
+
+    if (existing && existing.ws !== ws) {
+      existing.close('replaced by a newer desktop companion connection');
+    }
+
     return {
       connection,
       device: this.getDeviceRecordByDeviceId(userId, record.deviceId),
@@ -201,17 +217,21 @@ class DesktopCompanionRegistry {
 
   unregisterConnection(connection) {
     const userMap = this._getUserMap(connection.userId);
-    if (userMap && userMap.get(connection.deviceId) === connection) {
+    const isOwner = userMap != null && userMap.get(connection.deviceId) === connection;
+    if (isOwner) {
       userMap.delete(connection.deviceId);
       if (userMap.size === 0) {
         this.connectionsByUser.delete(String(connection.userId));
       }
+      // Only mark offline in the DB when this connection is still the active owner.
+      // If a newer connection has already taken over (reconnect race), its
+      // _upsertDeviceRecord already wrote status='online' and we must not clobber it.
+      this.db.prepare(
+        `UPDATE desktop_companion_devices
+         SET status = 'offline', updated_at = datetime('now')
+         WHERE user_id = ? AND device_id = ?`
+      ).run(connection.userId, connection.deviceId);
     }
-    this.db.prepare(
-      `UPDATE desktop_companion_devices
-       SET status = 'offline', updated_at = datetime('now')
-       WHERE user_id = ? AND device_id = ?`
-    ).run(connection.userId, connection.deviceId);
   }
 
   touchConnection(userId, deviceId, patch = {}) {
@@ -253,6 +273,11 @@ class DesktopCompanionRegistry {
     return this.getDeviceRecordByDeviceId(userId, deviceId);
   }
 
+  isConnected(userId) {
+    const userMap = this._getUserMap(userId);
+    return userMap != null && userMap.size > 0;
+  }
+
   getConnection(userId, deviceId) {
     const userMap = this._getUserMap(userId);
     if (!userMap) return null;
@@ -351,9 +376,19 @@ class DesktopCompanionRegistry {
   getStatus(userId) {
     const devices = this.listDevices(userId);
     const onlineDevices = devices.filter((device) => device.online);
+    let selectedDeviceId = this.getSelectedDeviceId(userId);
+
+    // Auto-select the most-recently-online device when there is no valid selection.
+    // listDevices returns online devices first, ordered by last_seen_at DESC.
+    const selectionIsOnline = selectedDeviceId && onlineDevices.some((d) => d.deviceId === selectedDeviceId);
+    if (!selectionIsOnline && onlineDevices.length > 0) {
+      selectedDeviceId = onlineDevices[0].deviceId;
+      this.setSelectedDeviceId(userId, selectedDeviceId);
+    }
+
     return {
       connected: onlineDevices.length > 0,
-      selectedDeviceId: this.getSelectedDeviceId(userId),
+      selectedDeviceId,
       onlineCount: onlineDevices.length,
       devices,
     };
@@ -446,6 +481,8 @@ class DesktopCompanionConnection {
         this.ws.close(1000, String(reason || 'closing').slice(0, 120));
       }
     } catch {}
+    // unregisterConnection is intentionally called here (not inside _closePending)
+    // so it runs synchronously before any async ws 'close' event can fire.
     this.registry.unregisterConnection(this);
     this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.'));
   }
@@ -512,6 +549,10 @@ class DesktopCompanionConnection {
       pending.reject(error);
     }
     this.pending.clear();
+    // unregisterConnection is idempotent (ownership-checked) so calling it
+    // here is safe whether we arrived via close(), the ws 'close' event, or
+    // both. It ensures natural socket drops (no explicit close() call) still
+    // mark the device offline.
     this.registry.unregisterConnection(this);
   }
 }

package/server/services/runtime/docker-vm-manager.js

@@ -102,17 +102,38 @@ const server = http.createServer(async (req, res) => {
 
     // ── CLI execution ──────────────────────────────────────────────────────
     if (req.method === 'POST' && url === '/exec') {
+      const timeoutMs = Math.min(Number(b.timeout) || 15 * 60 * 1000, 20 * 60 * 1000);
       const child = spawn('sh', ['-c', b.command || 'true'], {
         cwd: b.cwd || '/tmp',
         env: { ...process.env, ...b.env },
       });
       const pid = child.pid;
-      let stdout = '', stderr = '';
+      let stdout = '', stderr = '', settled = false;
       procs.set(pid, child);
       child.stdout.on('data', d => { stdout += d; });
       child.stderr.on('data', d => { stderr += d; });
-      child.on('close', code => { procs.delete(pid); json(res, { stdout, stderr, code: code ?? 1, pid }); });
-      child.on('error', err => { procs.delete(pid); json(res, { stdout, stderr, code: 1, pid, error: err.message }); });
+      const timer = setTimeout(() => {
+        if (settled) return;
+        settled = true;
+        procs.delete(pid);
+        try { child.kill('SIGKILL'); } catch {}
+        json(res, { stdout, stderr, exitCode: null, code: null, pid, timedOut: true, killed: true });
+      }, timeoutMs);
+      child.on('close', code => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        procs.delete(pid);
+        const exitCode = code ?? 1;
+        json(res, { stdout, stderr, exitCode, code: exitCode, pid, timedOut: false, killed: false });
+      });
+      child.on('error', err => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        procs.delete(pid);
+        json(res, { stdout, stderr, exitCode: 1, code: 1, pid, error: err.message, timedOut: false, killed: false });
+      });
       return;
     }
 
@@ -268,6 +289,7 @@ class DockerVMManager {
     this.image = options.image || CONTAINER_IMAGE;
     this.memoryMb = options.memoryMb || 2048;
     this.cpus = options.cpus || 2;
+    this.guestToken = String(options.guestToken || process.env.NEOAGENT_VM_GUEST_TOKEN || '').trim();
     this.#cleanupOrphans();
   }
 
@@ -309,6 +331,7 @@ class DockerVMManager {
       '--cpus', String(this.cpus),
       '-p', `127.0.0.1:${port}:${port}`,
       '-e', `AGENT_PORT=${port}`,
+      ...(this.guestToken ? ['-e', `NEOAGENT_VM_GUEST_TOKEN=${this.guestToken}`] : []),
       '--shm-size=2g',
       '--security-opt', 'no-new-privileges',
       '--label', CONTAINER_LABEL,

package/server/services/runtime/manager.js

@@ -100,9 +100,32 @@ class RuntimeManager {
   async getCliProviderForUser(userId) {
     const settings = this.getSettings(userId);
     if (settings.cli_backend === 'desktop' && this.hasActiveDesktopCompanion(userId)) {
-      return this.getDesktopCliProvider(userId);
+      const desktopProvider = this.getDesktopCliProvider(userId);
+      return {
+        backend: 'desktop-companion',
+        execute: (command, options = {}) => desktopProvider.executeCommand(command, options),
+        executeInteractive: (command, inputs = [], options = {}) => desktopProvider.executeCommand(command, { ...options, inputs }),
+        kill: () => Promise.resolve(false),
+      };
     }
-    return this.browserBackend.getCommandExecutorForUser(userId);
+    const executor = await this.browserBackend.getCommandExecutorForUser(userId);
+    return { ...executor, backend: 'vm' };
+  }
+
+  async executeCliCommand(userId, command, options = {}) {
+    const provider = await this.getCliProviderForUser(userId);
+    const result = await (options.pty === true && provider.executeInteractive
+      ? provider.executeInteractive(command, options.inputs || [], options)
+      : provider.execute(command, options));
+    return { ...result, backend: provider.backend };
+  }
+
+  getActiveBrowserBackend(userId) {
+    const settings = this.getSettings(userId);
+    if (settings.browser_backend === 'extension' && this.hasActiveExtensionBrowser(userId)) {
+      return 'extension';
+    }
+    return 'vm';
   }
 
   async getAndroidProviderForUser(userId) {