neoagent 2.4.0 → 2.4.1-beta.11

package/server/routes/browser.js

@@ -141,6 +141,20 @@ router.post('/click-point', async (req, res) => {
   }
 });
 
+router.post('/mouse-move', async (req, res) => {
+  try {
+    const { x, y } = req.body || {};
+    if (!Number.isFinite(Number(x)) || !Number.isFinite(Number(y))) {
+      return res.status(400).json({ error: 'x and y required' });
+    }
+    const bc = await getBrowserController(req);
+    const result = await bc.hoverPoint(x, y, req.body || {});
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
 // Fill form field
 router.post('/fill', async (req, res) => {
   try {

package/server/routes/browser_extension.js

@@ -4,10 +4,10 @@ const rateLimit = require('express-rate-limit');
 const { requireAuth } = require('../middleware/auth');
 const { sanitizeError } = require('../utils/security');
 const { createZipFromDirectory } = require('../services/browser/extension/zip');
+const { getExtensionManifest } = require('../services/browser/extension/manifest');
 
 const router = express.Router();
 const EXTENSION_DIR = path.join(__dirname, '..', '..', 'extensions', 'chrome-browser');
-const EXTENSION_MANIFEST = require('../../extensions/chrome-browser/manifest.json');
 const pairingLimiter = rateLimit({
   windowMs: 15 * 60 * 1000,
   max: 30,
@@ -55,10 +55,12 @@ router.post('/pairing/request', pairingLimiter, (req, res) => {
 });
 
 router.get('/latest', (req, res) => {
+  const manifest = getExtensionManifest();
   res.json({
-    name: EXTENSION_MANIFEST.name,
-    version: EXTENSION_MANIFEST.version,
-    minimumChromeVersion: EXTENSION_MANIFEST.minimum_chrome_version,
+    name: manifest.name,
+    version: manifest.version,
+    versionName: manifest.version_name,
+    minimumChromeVersion: manifest.minimum_chrome_version,
     downloadUrl: `${baseUrlFor(req)}/api/browser-extension/download`,
   });
 });
@@ -123,6 +125,19 @@ router.get('/status', requireAuth, (req, res) => {
   }
 });
 
+router.post('/select-token', requireAuth, (req, res) => {
+  try {
+    const registry = getRegistry(req);
+    const result = registry.setSelectedTokenId(req.session.userId, req.body?.tokenId);
+    res.json({
+      ...result,
+      status: registry.getStatus(req.session.userId),
+    });
+  } catch (err) {
+    res.status(err.status || 500).json({ error: sanitizeError(err) });
+  }
+});
+
 router.post('/revoke', requireAuth, (req, res) => {
   try {
     res.json(getRegistry(req).revoke(req.session.userId, req.body?.tokenId || null));
@@ -133,8 +148,10 @@ router.post('/revoke', requireAuth, (req, res) => {
 
 router.get('/download', requireAuth, (req, res) => {
   try {
+    const manifest = getExtensionManifest();
     const zip = createZipFromDirectory(EXTENSION_DIR, {
       overrides: {
+        'manifest.json': `${JSON.stringify(manifest, null, 2)}\n`,
         'config.mjs': extensionConfigFor(req),
       },
     });

package/server/routes/desktop.js

@@ -163,6 +163,16 @@ router.post('/click', (req, res) =>
     },
   )));
 
+router.post('/mouse-move', (req, res) =>
+  handleDesktopAction(req, res, (provider, request) => provider.mouseMove(
+    parseFiniteNumber(request.body?.x, 'x', { min: -100000, max: 100000 }),
+    parseFiniteNumber(request.body?.y, 'y', { min: -100000, max: 100000 }),
+    {
+      ...(request.body || {}),
+      deviceId: parseDeviceId(request.body?.deviceId),
+    },
+  )));
+
 router.post('/drag', (req, res) =>
   handleDesktopAction(req, res, (provider, request) => provider.drag({
     ...(request.body || {}),

package/server/routes/settings.js

@@ -297,7 +297,11 @@ router.put('/', async (req, res) => {
     'runtime_profile' in normalizedBody
     || 'runtime_backend' in normalizedBody
     || 'browser_backend' in normalizedBody
+    || 'browser_extension_token_id' in normalizedBody
+    || 'selected_browser_extension_token_id' in normalizedBody
     || 'android_backend' in normalizedBody
+    || 'cli_backend' in normalizedBody
+    || 'cli_desktop_device_id' in normalizedBody
     || 'mcp_backend' in normalizedBody
   ) {
     const validation = validateRuntimeSettings({

package/server/routes/stream.js

@@ -0,0 +1,187 @@
+'use strict';
+
+const express = require('express');
+const { requireAuth } = require('../middleware/auth');
+const { sanitizeError } = require('../utils/security');
+const { AndroidStream } = require('../services/streaming/android-stream');
+const { BrowserStream } = require('../services/streaming/browser-stream');
+
+const router = express.Router();
+router.use(requireAuth);
+
+function boundedInt(value, fallback, min, max) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.min(max, Math.max(min, Math.floor(parsed)));
+}
+
+function streamHub(req) {
+  const hub = req.app?.locals?.streamHub;
+  if (!hub) throw new Error('Stream hub is unavailable.');
+  return hub;
+}
+
+async function desktopProvider(req) {
+  const factory = req.app?.locals?.getDesktopProviderForUser;
+  if (typeof factory !== 'function') {
+    throw new Error('Desktop provider is unavailable.');
+  }
+  return factory(req.session?.userId);
+}
+
+async function androidController(req) {
+  const runtimeManager = req.app?.locals?.runtimeManager;
+  if (runtimeManager && typeof runtimeManager.getAndroidProviderForUser === 'function') {
+    return runtimeManager.getAndroidProviderForUser(req.session?.userId);
+  }
+  throw new Error('Android controller is unavailable.');
+}
+
+async function browserController(req) {
+  const runtimeManager = req.app?.locals?.runtimeManager;
+  if (runtimeManager && typeof runtimeManager.getBrowserProviderForUser === 'function') {
+    return runtimeManager.getBrowserProviderForUser(req.session?.userId);
+  }
+  throw new Error('Browser controller is unavailable.');
+}
+
+function normalizePlatform(value) {
+  const platform = String(value || '').trim().toLowerCase();
+  if (platform === 'desktop' || platform === 'android' || platform === 'browser') return platform;
+  const error = new Error('platform must be desktop, android, or browser.');
+  error.status = 400;
+  throw error;
+}
+
+function normalizeDeviceId(platform, value) {
+  const deviceId = String(value || '').trim();
+  if (deviceId) return deviceId;
+  if (platform === 'browser') return 'browser';
+  const error = new Error('deviceId is required.');
+  error.status = 400;
+  throw error;
+}
+
+async function resolveStartDeviceId(platform, req) {
+  const rawDeviceId = String(req.body?.deviceId || '').trim();
+  if (platform === 'browser') return { deviceId: 'browser', controller: null };
+  if (platform === 'android') {
+    const controller = await androidController(req);
+    const status = typeof controller.getStatus === 'function'
+      ? await controller.getStatus().catch(() => ({}))
+      : {};
+    const adbSerial = String(status?.adbSerial || controller.adbSerial || '').trim();
+    if (rawDeviceId && adbSerial && rawDeviceId !== adbSerial) {
+      const error = new Error('Android deviceId does not match the active emulator serial.');
+      error.status = 400;
+      throw error;
+    }
+    if (rawDeviceId) return { deviceId: rawDeviceId, controller };
+    if (!adbSerial) throw new Error('Android deviceId is required when no emulator serial is available.');
+    return { deviceId: adbSerial, controller };
+  }
+  if (rawDeviceId) return { deviceId: rawDeviceId, controller: null };
+  return { deviceId: normalizeDeviceId(platform, rawDeviceId), controller: null };
+}
+
+async function resolveStopDeviceId(platform, req) {
+  const rawDeviceId = String(req.body?.deviceId || '').trim();
+  if (rawDeviceId) return rawDeviceId;
+  if (platform === 'browser') return 'browser';
+  if (platform === 'android') {
+    const controller = await androidController(req);
+    const status = typeof controller.getStatus === 'function'
+      ? await controller.getStatus().catch(() => ({}))
+      : {};
+    const adbSerial = String(status?.adbSerial || controller.adbSerial || '').trim();
+    if (adbSerial) return adbSerial;
+  }
+  const error = new Error('deviceId is required.');
+  error.status = 400;
+  throw error;
+}
+
+function sendError(res, err) {
+  res.status(Number(err?.status || 500)).json({ error: sanitizeError(err) });
+}
+
+router.post('/start', async (req, res) => {
+  try {
+    const userId = req.session?.userId;
+    const platform = normalizePlatform(req.body?.platform);
+    const resolved = await resolveStartDeviceId(platform, req);
+    const deviceId = resolved.deviceId;
+    const fps = boundedInt(req.body?.fps, platform === 'android' ? 12 : 15, 1, platform === 'android' ? 30 : 20);
+    const quality = boundedInt(req.body?.quality, platform === 'android' ? 75 : 80, 30, 95);
+    const hub = streamHub(req);
+    await hub.stopStream(userId, platform, deviceId, 'restart');
+
+    if (platform === 'desktop') {
+      const provider = await desktopProvider(req);
+      const result = await provider.startStream({
+        deviceId,
+        fps,
+        quality,
+        displayId: req.body?.displayId || null,
+      });
+      const resolvedDeviceId = result?.deviceId || result?.device?.deviceId || deviceId;
+      hub.markStarted(userId, resolvedDeviceId, platform, { fps, quality }, async () => {
+        try {
+          await provider.stopStream({ deviceId: resolvedDeviceId });
+        } catch (error) {
+          console.error('[StreamRoute] failed to stop desktop stream', {
+            userId,
+            deviceId: resolvedDeviceId,
+            error: String(error?.message || error),
+          });
+        }
+      });
+      return res.json({ ok: true, platform, deviceId: resolvedDeviceId, fps, quality });
+    }
+
+    if (platform === 'android') {
+      const controller = resolved.controller || await androidController(req);
+      const stream = new AndroidStream({ userId, deviceId, controller, streamHub: hub, fps, quality });
+      stream.start();
+      hub.markStarted(userId, deviceId, platform, { fps, quality }, () => stream.stop());
+      return res.json({ ok: true, platform, deviceId, fps, quality });
+    }
+
+    const controller = await browserController(req);
+    const stream = new BrowserStream({ userId, deviceId, controller, streamHub: hub, fps, quality });
+    stream.start();
+    hub.markStarted(userId, deviceId, platform, { fps, quality }, () => stream.stop());
+    return res.json({ ok: true, platform, deviceId, fps, quality });
+  } catch (err) {
+    sendError(res, err);
+  }
+});
+
+router.post('/stop', async (req, res) => {
+  try {
+    const platform = normalizePlatform(req.body?.platform);
+    const deviceId = await resolveStopDeviceId(platform, req);
+    const stopped = await streamHub(req).stopStream(req.session?.userId, platform, deviceId, 'api_stop');
+    res.json({ ok: true, stopped, platform, deviceId });
+  } catch (err) {
+    sendError(res, err);
+  }
+});
+
+router.get('/status', async (req, res) => {
+  try {
+    const platform = req.query?.platform ? normalizePlatform(req.query.platform) : null;
+    const deviceId = req.query?.deviceId
+      ? normalizeDeviceId(platform || 'desktop', req.query.deviceId)
+      : null;
+    const hub = streamHub(req);
+    if (deviceId) {
+      return res.json(hub.status(req.session?.userId, platform || 'desktop', deviceId));
+    }
+    return res.json({ streams: hub.listStatus(req.session?.userId), platform });
+  } catch (err) {
+    sendError(res, err);
+  }
+});
+
+module.exports = router;

package/server/services/ai/tools.js

@@ -292,7 +292,7 @@ function getAvailableTools(app, options = {}) {
     const tools = [
         {
             name: 'execute_command',
-            description: 'Execute a terminal/shell command as a normal recoverable agent step. Waits for the process to exit, supports PTY for interactive programs, and returns stdout, stderr, exit code, timeout state, and duration so later reasoning can inspect failures, install missing dependencies, and retry when needed.',
+            description: 'Execute a terminal/shell command as a normal recoverable agent step. Waits for the process to exit, supports PTY for interactive programs, and returns stdout, stderr, exit code, timeout state, duration, and a backend field ("vm" or "desktop-companion") indicating where the command ran. Commands run inside the isolated VM unless the cli_backend setting is set to "desktop" and a companion app is connected, in which case the command runs on the companion desktop machine.',
             parameters: {
                 type: 'object',
                 properties: {
@@ -308,7 +308,7 @@ function getAvailableTools(app, options = {}) {
         },
         {
             name: 'browser_navigate',
-            description: 'Navigate the browser to a URL and return page content/screenshot',
+            description: 'Navigate the browser to a URL and return page content/screenshot. The result includes a backend field ("vm" or "extension") indicating whether the VM browser or the paired browser extension handled the request.',
             parameters: {
                 type: 'object',
                 properties: {
@@ -1401,10 +1401,13 @@ async function executeTool(toolName, args, context, engine) {
         allowMultipleProactiveMessages = false
     } = context;
     const runtime = () => app?.locals?.runtimeManager || engine.runtimeManager || null;
-    const bc = () => {
+    const bc = async () => {
         const manager = runtime();
         if (manager && typeof manager.getBrowserProviderForUser === 'function') {
-            return manager.getBrowserProviderForUser(userId);
+            const backend = typeof manager.getActiveBrowserBackend === 'function'
+                ? await Promise.resolve(manager.getActiveBrowserBackend(userId))
+                : 'vm';
+            return { provider: await manager.getBrowserProviderForUser(userId), backend };
         }
         throw new Error('Browser provider is unavailable. VM runtime is required.');
     };
@@ -1478,59 +1481,67 @@ async function executeTool(toolName, args, context, engine) {
 
         case 'execute_command': {
             const runtimeManager = runtime();
-            if (!runtimeManager || typeof runtimeManager.executeCommand !== 'function') {
-                return { error: 'Command execution is unavailable. VM runtime is required.' };
+            if (!runtimeManager) {
+                return { error: 'Command execution is unavailable. No runtime manager found.' };
             }
-            return await runtimeManager.executeCommand(userId, args.command, {
+            const execOptions = {
                 cwd: args.cwd,
                 timeout: args.timeout || (args.pty ? 20 * 60 * 1000 : 15 * 60 * 1000),
                 stdinInput: args.stdin_input,
                 pty: args.pty === true,
-                inputs: args.inputs || [],
-            });
+                inputs: Array.isArray(args.inputs) ? args.inputs : [],
+            };
+            if (typeof runtimeManager.executeCliCommand === 'function') {
+                return await runtimeManager.executeCliCommand(userId, args.command, execOptions);
+            }
+            // Legacy fallback — older runtime manager without CLI routing.
+            if (typeof runtimeManager.executeCommand !== 'function') {
+                return { error: 'Command execution is unavailable. VM runtime is required.' };
+            }
+            return { ...await runtimeManager.executeCommand(userId, args.command, execOptions), backend: 'vm' };
         }
 
         case 'browser_navigate': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.navigate(args.url, {
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.navigate(args.url, {
                 screenshot: args.screenshot !== false,
                 waitFor: args.waitFor,
                 fullPage: args.fullPage
-            });
+            }), backend };
         }
 
         case 'browser_click': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.click(args.selector, args.text, args.screenshot !== false);
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.click(args.selector, args.text, args.screenshot !== false), backend };
         }
 
         case 'browser_type': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.type(args.selector, args.text, {
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.type(args.selector, args.text, {
                 clear: args.clear !== false,
                 pressEnter: args.pressEnter
-            });
+            }), backend };
         }
 
         case 'browser_extract': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.extract(args.selector, args.attribute, args.all);
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.extract(args.selector, args.attribute, args.all), backend };
         }
 
         case 'browser_screenshot': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.screenshot({ fullPage: args.fullPage, selector: args.selector });
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.screenshot({ fullPage: args.fullPage, selector: args.selector }), backend };
         }
 
         case 'browser_evaluate': {
-            const controller = await bc();
-            if (!controller) return { error: 'Browser controller not available' };
-            return await controller.evaluate(args.script);
+            const { provider, backend } = await bc();
+            if (!provider) return { error: 'Browser controller not available' };
+            return { ...await provider.evaluate(args.script), backend };
         }
 
         case 'android_start_emulator': {

package/server/services/android/controller.js

@@ -347,12 +347,16 @@ class AndroidController {
   // ── Actions ───────────────────────────────────────────────────────────────
 
   async screenshot(_opts = {}) {
-    const serial = this.#requireSerial();
-    const r = this.#adbCapture(serial, ['exec-out', 'screencap', '-p']);
+    const r = await this.capturePng();
     if (!r?.length) throw new Error('screencap returned no data');
     return { screenshotPath: this.#saveArtifact(r) };
   }
 
+  async capturePng(_opts = {}) {
+    const serial = this.#requireSerial();
+    return this.#adbCaptureAsync(serial, ['exec-out', 'screencap', '-p']);
+  }
+
   async observe(_opts = {}) { return this.screenshot(); }
 
   async tap({ x, y } = {}) {
@@ -469,6 +473,41 @@ class AndroidController {
     return (r.status === 0 && r.stdout?.length) ? r.stdout : null;
   }
 
+  #adbCaptureAsync(serial, args) {
+    return new Promise((resolve, reject) => {
+      const proc = spawn(adbBin(this.sdkDir), ['-s', serial, ...args]);
+      const chunks = [];
+      const errors = [];
+      let settled = false;
+      const timer = setTimeout(() => {
+        if (settled) return;
+        settled = true;
+        try { proc.kill('SIGKILL'); } catch {}
+        reject(new Error('adb capture timed out'));
+      }, 15000);
+      timer.unref?.();
+      proc.stdout?.on('data', (chunk) => chunks.push(chunk));
+      proc.stderr?.on('data', (chunk) => errors.push(chunk));
+      proc.on('error', (error) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        reject(error);
+      });
+      proc.on('close', (code) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        if (code === 0 && chunks.length) {
+          resolve(Buffer.concat(chunks));
+          return;
+        }
+        const message = Buffer.concat(errors).toString('utf8').trim();
+        reject(new Error(message || `adb capture exit ${code}`));
+      });
+    });
+  }
+
   #saveArtifact(data) {
     if (!data || !this.artifactStore) return null;
     const alloc = this.artifactStore.allocateFile(this.userId, { kind: 'screenshot', extension: 'png', contentType: 'image/png' });

package/server/services/browser/controller.js

@@ -453,6 +453,22 @@ class BrowserController {
     };
   }
 
+  async screenshotJpeg(quality = 80, options = {}) {
+    const page = await this.ensurePage();
+    const screenshotOptions = {
+      type: 'jpeg',
+      quality: Math.min(95, Math.max(30, Math.floor(Number(quality) || 80))),
+      fullPage: options.fullPage === true,
+    };
+    if (options.selector) {
+      const element = await page.$(options.selector);
+      if (element) {
+        return element.screenshot(screenshotOptions);
+      }
+    }
+    return page.screenshot(screenshotOptions);
+  }
+
   async navigate(url, options = {}) {
     const page = await this.ensurePage();
 
@@ -584,6 +600,24 @@ class BrowserController {
     }
   }
 
+  async hoverPoint(x, y, options = {}) {
+    const page = await this.ensurePage();
+    try {
+      const px = Math.max(0, Math.round(Number(x) || 0));
+      const py = Math.max(0, Math.round(Number(y) || 0));
+      await page.mouse.move(px, py, { steps: options.steps || 1 });
+      return {
+        success: true,
+        x: px,
+        y: py,
+        url: page.url(),
+        title: await page.title()
+      };
+    } catch (err) {
+      return { error: err.message };
+    }
+  }
+
   async scroll(deltaX = 0, deltaY = 0, screenshot = true) {
     const page = await this.ensurePage();
 

package/server/services/browser/extension/manifest.js

@@ -0,0 +1,33 @@
+'use strict';
+
+const BASE_MANIFEST = require('../../../../extensions/chrome-browser/manifest.json');
+const { getVersionInfo } = require('../../../utils/version');
+
+function chromeVersionFor(packageVersion) {
+  const parts = String(packageVersion || '')
+    .match(/\d+/g)
+    ?.slice(0, 4)
+    .map((part) => Math.min(Number(part) || 0, 65535)) || [];
+
+  while (parts.length < 3) {
+    parts.push(0);
+  }
+
+  return parts.join('.');
+}
+
+function getExtensionManifest() {
+  const { packageVersion } = getVersionInfo();
+  const displayVersion = packageVersion || BASE_MANIFEST.version_name || BASE_MANIFEST.version;
+
+  return {
+    ...BASE_MANIFEST,
+    version: chromeVersionFor(displayVersion),
+    version_name: displayVersion,
+  };
+}
+
+module.exports = {
+  chromeVersionFor,
+  getExtensionManifest,
+};

package/server/services/browser/extension/provider.js

@@ -17,6 +17,7 @@ class ExtensionBrowserProvider {
   constructor(options = {}) {
     this.registry = options.registry;
     this.userId = options.userId != null ? String(options.userId) : null;
+    this.tokenId = options.tokenId ? String(options.tokenId) : null;
     this.artifactStore = options.artifactStore || null;
     this.headless = false;
   }
@@ -29,13 +30,16 @@ class ExtensionBrowserProvider {
 
   async #dispatch(command, payload = {}, options = {}) {
     this.#assertReady();
-    const result = await this.registry.dispatch(this.userId, command, payload, options);
+    const result = await this.registry.dispatch(this.userId, command, payload, {
+      ...options,
+      tokenId: options.tokenId || this.tokenId,
+    });
     return this.#materialize(result);
   }
 
   #disconnect() {
     if (!this.registry || this.userId == null) return;
-    const connection = this.registry.getConnection(this.userId);
+    const connection = this.registry.getConnection(this.userId, this.tokenId);
     if (connection) {
       connection.close('browser extension provider closed');
     }
@@ -131,7 +135,7 @@ class ExtensionBrowserProvider {
   }
 
   async closeBrowser() {
-    if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId)) {
+    if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId, this.tokenId)) {
       return { success: true, extensionConnected: false };
     }
     const result = await this.#dispatch(EXTENSION_COMMANDS.CLOSE, {});
@@ -152,14 +156,16 @@ class ExtensionBrowserProvider {
   }
 
   async getPageInfo() {
-    if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId)) {
+    if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId, this.tokenId)) {
       return { url: null, title: null, extensionConnected: false };
     }
-    return this.registry.dispatch(this.userId, EXTENSION_COMMANDS.GET_PAGE_INFO, {});
+    return this.registry.dispatch(this.userId, EXTENSION_COMMANDS.GET_PAGE_INFO, {}, {
+      tokenId: this.tokenId,
+    });
   }
 
   isLaunched() {
-    return Boolean(this.registry && this.userId != null && this.registry.isConnected(this.userId));
+    return Boolean(this.registry && this.userId != null && this.registry.isConnected(this.userId, this.tokenId));
   }
 
   getPageCount() {