neoagent 2.0.7 → 2.1.0

package/server/routes/android.js

@@ -0,0 +1,136 @@
+const express = require('express');
+const router = express.Router();
+const { requireAuth } = require('../middleware/auth');
+const { sanitizeError } = require('../utils/security');
+
+router.use(requireAuth);
+
+router.get('/status', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.getStatus());
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/start', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.startEmulator(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/stop', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.stopEmulator());
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.get('/devices', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json({ devices: await controller.listDevices() });
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/screenshot', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.screenshot(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/ui-dump', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.dumpUi(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.get('/apps', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.listApps({
+      includeSystem: req.query.includeSystem === 'true',
+    }));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/open-app', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.openApp(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/open-intent', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.openIntent(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/tap', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.tap(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/type', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.type(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/swipe', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.swipe(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/press-key', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.pressKey(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/wait-for', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.waitFor(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+module.exports = router;

package/server/routes/browser.js

@@ -6,13 +6,19 @@ const { sanitizeError } = require('../utils/security');
 router.use(requireAuth);
 
 // Get browser status
-router.get('/status', (req, res) => {
-  const bc = req.app.locals.browserController;
-  res.json({
-    launched: bc.isLaunched(),
-    pages: bc.getPageCount(),
-    headless: bc.headless
-  });
+router.get('/status', async (req, res) => {
+  try {
+    const bc = req.app.locals.browserController;
+    const pageInfo = await bc.getPageInfo();
+    res.json({
+      launched: bc.isLaunched(),
+      pages: bc.getPageCount(),
+      headless: bc.headless,
+      pageInfo,
+    });
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
 });
 
 // Launch browser
@@ -56,7 +62,21 @@ router.post('/click', async (req, res) => {
   try {
     const { selector, text } = req.body;
     const bc = req.app.locals.browserController;
-    const result = await bc.click(selector, { text });
+    const result = await bc.click(selector, text, req.body?.screenshot !== false);
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/click-point', async (req, res) => {
+  try {
+    const { x, y } = req.body || {};
+    if (!Number.isFinite(Number(x)) || !Number.isFinite(Number(y))) {
+      return res.status(400).json({ error: 'x and y required' });
+    }
+    const bc = req.app.locals.browserController;
+    const result = await bc.clickPoint(x, y, req.body?.screenshot !== false);
     res.json(result);
   } catch (err) {
     res.status(500).json({ error: sanitizeError(err) });
@@ -70,7 +90,51 @@ router.post('/fill', async (req, res) => {
     if (!selector || value === undefined) return res.status(400).json({ error: 'selector and value required' });
 
     const bc = req.app.locals.browserController;
-    const result = await bc.fill(selector, value);
+    const result = await bc.type(selector, String(value), {
+      clear: req.body?.clear !== false,
+      pressEnter: req.body?.pressEnter === true,
+      screenshot: req.body?.screenshot !== false,
+    });
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/type-text', async (req, res) => {
+  try {
+    const { text } = req.body || {};
+    const bc = req.app.locals.browserController;
+    const result = await bc.typeText(String(text || ''), {
+      pressEnter: req.body?.pressEnter === true,
+      screenshot: req.body?.screenshot !== false,
+    });
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/press-key', async (req, res) => {
+  try {
+    const { key } = req.body || {};
+    if (!key) return res.status(400).json({ error: 'key required' });
+    const bc = req.app.locals.browserController;
+    const result = await bc.pressKey(key, req.body?.screenshot !== false);
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/scroll', async (req, res) => {
+  try {
+    const bc = req.app.locals.browserController;
+    const result = await bc.scroll(
+      req.body?.deltaX ?? 0,
+      req.body?.deltaY ?? 0,
+      req.body?.screenshot !== false,
+    );
     res.json(result);
   } catch (err) {
     res.status(500).json({ error: sanitizeError(err) });

package/server/routes/recordings.js

@@ -110,4 +110,24 @@ router.post('/:sessionId/retry', async (req, res) => {
   }
 });
 
+router.delete('/:sessionId/segments/:segmentId', (req, res) => {
+  try {
+    const manager = req.app.locals.recordingManager;
+    const session = manager.deleteTranscriptSegment(
+      req.session.userId,
+      req.params.sessionId,
+      req.params.segmentId,
+    );
+    res.json({ session });
+  } catch (err) {
+    const message = sanitizeError(err);
+    const status = /not found/i.test(message)
+      ? 404
+      : /positive integer/i.test(message)
+        ? 400
+        : 500;
+    res.status(status).json({ error: message });
+  }
+});
+
 module.exports = router;

package/server/routes/scheduler.js

@@ -14,13 +14,13 @@ router.get('/', (req, res) => {
 // Create a new scheduled task
 router.post('/', (req, res) => {
   try {
-    const { name, cronExpression, prompt, enabled } = req.body;
+    const { name, cronExpression, prompt, enabled, model } = req.body;
     if (!name || !cronExpression || !prompt) {
       return res.status(400).json({ error: 'name, cronExpression, and prompt required' });
     }
 
     const scheduler = req.app.locals.scheduler;
-    const task = scheduler.createTask(req.session.userId, { name, cronExpression, prompt, enabled });
+    const task = scheduler.createTask(req.session.userId, { name, cronExpression, prompt, enabled, model });
     res.status(201).json(task);
   } catch (err) {
     res.status(400).json({ error: sanitizeError(err) });

package/server/routes/settings.js

@@ -240,6 +240,8 @@ router.post('/update', (req, res) => {
     message: 'Launching update job',
     startedAt: new Date().toISOString(),
     completedAt: null,
+    versionBefore: null,
+    versionAfter: null,
     changelog: [],
     logs: []
   });

package/server/services/ai/engine.js

@@ -115,6 +115,7 @@ class AgentEngine {
     this.maxIterations = 12;
     this.activeRuns = new Map();
     this.browserController = services.browserController || null;
+    this.androidController = services.androidController || null;
     this.messagingManager = services.messagingManager || null;
     this.mcpManager = services.mcpManager || services.mcpClient || null;
     this.skillRunner = services.skillRunner || null;
@@ -587,6 +588,7 @@ class AgentEngine {
 
   getStepType(toolName) {
     if (toolName.startsWith('browser_')) return 'browser';
+    if (toolName.startsWith('android_')) return 'android';
     if (toolName === 'execute_command') return 'cli';
     if (toolName.startsWith('memory_')) return 'memory';
     if (toolName === 'send_message') return 'messaging';

package/server/services/ai/toolResult.js

@@ -85,6 +85,25 @@ function compactToolResult(toolName, toolArgs = {}, toolResult, options = {}) {
       });
       break;
 
+    case 'android_dump_ui':
+      envelope = trimObject({
+        tool: toolName,
+        serial: toolResult?.serial,
+        nodeCount: toolResult?.nodeCount,
+        uiDumpPath: toolResult?.uiDumpPath,
+        preview: clampText(JSON.stringify(toolResult?.preview || []).slice(0, Math.floor(softLimit * 0.55)), Math.floor(softLimit * 0.55))
+      });
+      break;
+
+    case 'android_list_apps':
+      envelope = trimObject({
+        tool: toolName,
+        serial: toolResult?.serial,
+        count: toolResult?.count,
+        preview: lineExcerpt((toolResult?.packages || []).slice(0, 20).join('\n'), 20, Math.floor(softLimit * 0.6))
+      });
+      break;
+
     case 'http_request':
       envelope = trimObject({
         tool: toolName,

package/server/services/ai/tools.js

@@ -142,6 +142,177 @@ function getAvailableTools(app, options = {}) {
                 required: ['script']
             }
         },
+        {
+            name: 'android_start_emulator',
+            description: 'Bootstrap Android tools if needed and start the managed Android emulator.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    headless: { type: 'boolean', description: 'Run the emulator headless (default true)' },
+                    timeoutMs: { type: 'number', description: 'Boot timeout in milliseconds (default 240000)' }
+                }
+            }
+        },
+        {
+            name: 'android_stop_emulator',
+            description: 'Stop the managed Android emulator.',
+            parameters: {
+                type: 'object',
+                properties: {}
+            }
+        },
+        {
+            name: 'android_list_devices',
+            description: 'List ADB-connected Android devices and emulators.',
+            parameters: {
+                type: 'object',
+                properties: {}
+            }
+        },
+        {
+            name: 'android_open_app',
+            description: 'Open an installed Android app by package name, optionally with a specific activity.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    packageName: { type: 'string', description: 'Android package name, e.g. com.google.android.apps.maps' },
+                    activity: { type: 'string', description: 'Optional activity name to launch' }
+                },
+                required: ['packageName']
+            }
+        },
+        {
+            name: 'android_open_intent',
+            description: 'Open an Android intent for deep links, navigation, messaging, or app-specific actions.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    action: { type: 'string', description: 'Intent action, e.g. android.intent.action.VIEW' },
+                    dataUri: { type: 'string', description: 'Intent data URI, e.g. geo:0,0?q=coffee or smsto:+1234567890' },
+                    packageName: { type: 'string', description: 'Optional package name to target' },
+                    component: { type: 'string', description: 'Optional fully qualified component name' },
+                    mimeType: { type: 'string', description: 'Optional MIME type' },
+                    extras: { type: 'object', description: 'Optional string extras added via --es' }
+                }
+            }
+        },
+        {
+            name: 'android_tap',
+            description: 'Tap the Android screen at coordinates or by matching a UI element from the current UI dump.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    x: { type: 'number', description: 'Absolute X coordinate' },
+                    y: { type: 'number', description: 'Absolute Y coordinate' },
+                    text: { type: 'string', description: 'Visible text to match in the UI dump' },
+                    resourceId: { type: 'string', description: 'Android resource-id to match' },
+                    description: { type: 'string', description: 'content-desc / accessibility label to match' },
+                    className: { type: 'string', description: 'Optional class name filter' },
+                    packageName: { type: 'string', description: 'Optional package filter' },
+                    clickable: { type: 'boolean', description: 'Prefer clickable elements' }
+                }
+            }
+        },
+        {
+            name: 'android_type',
+            description: 'Type text into the focused Android field, optionally tapping a matched element first.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    text: { type: 'string', description: 'Text to type' },
+                    textSelector: { type: 'string', description: 'Visible text of the field to focus first' },
+                    resourceId: { type: 'string', description: 'resource-id of the field to focus first' },
+                    description: { type: 'string', description: 'content-desc of the field to focus first' },
+                    className: { type: 'string', description: 'Optional class filter when focusing an element' },
+                    clear: { type: 'boolean', description: 'Attempt to clear before typing' },
+                    pressEnter: { type: 'boolean', description: 'Press Enter after typing' }
+                },
+                required: ['text']
+            }
+        },
+        {
+            name: 'android_swipe',
+            description: 'Swipe across the Android screen using absolute coordinates.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    x1: { type: 'number', description: 'Start X coordinate' },
+                    y1: { type: 'number', description: 'Start Y coordinate' },
+                    x2: { type: 'number', description: 'End X coordinate' },
+                    y2: { type: 'number', description: 'End Y coordinate' },
+                    durationMs: { type: 'number', description: 'Swipe duration in milliseconds (default 300)' }
+                },
+                required: ['x1', 'y1', 'x2', 'y2']
+            }
+        },
+        {
+            name: 'android_press_key',
+            description: 'Send an Android key event such as home, back, enter, menu, app_switch, or a numeric key code.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    key: { type: 'string', description: 'Named key or numeric Android key code' }
+                },
+                required: ['key']
+            }
+        },
+        {
+            name: 'android_wait_for',
+            description: 'Poll Android UI dumps until a matching element appears.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    text: { type: 'string', description: 'Visible text to wait for' },
+                    resourceId: { type: 'string', description: 'resource-id to wait for' },
+                    description: { type: 'string', description: 'content-desc to wait for' },
+                    className: { type: 'string', description: 'Optional class filter' },
+                    packageName: { type: 'string', description: 'Optional package filter' },
+                    clickable: { type: 'boolean', description: 'Require the matched element to be clickable' },
+                    timeoutMs: { type: 'number', description: 'Timeout in milliseconds (default 20000)' },
+                    intervalMs: { type: 'number', description: 'Polling interval in milliseconds (default 1500)' },
+                    screenshot: { type: 'boolean', description: 'Capture a screenshot after a match (default true)' }
+                }
+            }
+        },
+        {
+            name: 'android_dump_ui',
+            description: 'Capture the current Android UIAutomator XML dump and return a preview of the nodes.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    includeNodes: { type: 'boolean', description: 'Include a preview of the parsed nodes (default true)' }
+                }
+            }
+        },
+        {
+            name: 'android_screenshot',
+            description: 'Capture a screenshot from the active Android device or emulator.',
+            parameters: {
+                type: 'object',
+                properties: {}
+            }
+        },
+        {
+            name: 'android_list_apps',
+            description: 'List installed Android app package names.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    includeSystem: { type: 'boolean', description: 'Include system apps (default false)' }
+                }
+            }
+        },
+        {
+            name: 'android_install_apk',
+            description: 'Install or replace an APK on the Android emulator.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    apkPath: { type: 'string', description: 'Absolute path to the APK file on disk' }
+                },
+                required: ['apkPath']
+            }
+        },
         {
             name: 'web_search',
             description: 'Search the public web without opening the browser. Uses Brave Search API for fast result retrieval.',
@@ -437,6 +608,7 @@ function getAvailableTools(app, options = {}) {
                     cron_expression: { type: 'string', description: 'Cron expression for the schedule, e.g. "0 9 * * 1-5" for weekdays at 9am, "*/30 * * * *" for every 30 minutes. Use standard 5-field cron syntax.' },
                     prompt: { type: 'string', description: 'The prompt/instructions the agent will run when triggered. Be specific about what to do and who to notify.' },
                     enabled: { type: 'boolean', description: 'Whether to activate immediately (default true)' },
+                    model: { type: 'string', description: 'Optional specific AI model ID to force for this task. Omit to use the normal automatic/default model selection.' },
                     call_to: { type: 'string', description: 'E.164 phone number to call via Telnyx when this task fires, e.g. "+12125550100".' },
                     call_greeting: { type: 'string', description: 'Opening sentence spoken to the user when the call is answered. Required if call_to is set.' }
                 },
@@ -452,6 +624,7 @@ function getAvailableTools(app, options = {}) {
                     name: { type: 'string', description: 'Short descriptive name, e.g. "Remind about meeting"' },
                     run_at: { type: 'string', description: 'ISO 8601 datetime when the run should fire, e.g. "2026-03-09T22:00:00"' },
                     prompt: { type: 'string', description: 'The prompt/instructions the agent will execute at that time. Be specific.' },
+                    model: { type: 'string', description: 'Optional specific AI model ID to force for this run. Omit to use the normal automatic/default model selection.' },
                     call_to: { type: 'string', description: 'Optional E.164 phone number to call via Telnyx when this fires.' },
                     call_greeting: { type: 'string', description: 'Opening sentence spoken when the Telnyx call is answered.' }
                 },
@@ -485,6 +658,7 @@ function getAvailableTools(app, options = {}) {
                     cron_expression: { type: 'string', description: 'New cron expression, e.g. "0 8 * * *" for daily at 8am' },
                     prompt: { type: 'string', description: 'New prompt/instructions for the task' },
                     enabled: { type: 'boolean', description: 'Enable or disable the task' },
+                    model: { type: 'string', description: 'Specific AI model ID for this task. Set to empty string to clear the override and go back to automatic/default selection.' },
                     call_to: { type: 'string', description: 'E.164 phone number to call via Telnyx when this task fires. Set to empty string to remove.' },
                     call_greeting: { type: 'string', description: 'New opening sentence spoken when the Telnyx call is answered.' }
                 },
@@ -600,6 +774,7 @@ function getAvailableTools(app, options = {}) {
 async function executeTool(toolName, args, context, engine) {
     const { userId, runId, app } = context;
     const bc = () => app?.locals?.browserController || engine.browserController;
+    const ac = () => app?.locals?.androidController || engine.androidController;
     const msg = () => app?.locals?.messagingManager || engine.messagingManager;
     const mcp = () => app?.locals?.mcpManager || app?.locals?.mcpClient || engine.mcpManager;
     const sk = () => app?.locals?.skillRunner || engine.skillRunner;
@@ -665,6 +840,90 @@ async function executeTool(toolName, args, context, engine) {
             return await controller.evaluate(args.script);
         }
 
+        case 'android_start_emulator': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.startEmulator(args || {});
+        }
+
+        case 'android_stop_emulator': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.stopEmulator();
+        }
+
+        case 'android_list_devices': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return { devices: await controller.listDevices() };
+        }
+
+        case 'android_open_app': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.openApp(args || {});
+        }
+
+        case 'android_open_intent': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.openIntent(args || {});
+        }
+
+        case 'android_tap': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.tap(args || {});
+        }
+
+        case 'android_type': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.type(args || {});
+        }
+
+        case 'android_swipe': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.swipe(args || {});
+        }
+
+        case 'android_press_key': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.pressKey(args || {});
+        }
+
+        case 'android_wait_for': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.waitFor(args || {});
+        }
+
+        case 'android_dump_ui': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.dumpUi(args || {});
+        }
+
+        case 'android_screenshot': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.screenshot(args || {});
+        }
+
+        case 'android_list_apps': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.listApps(args || {});
+        }
+
+        case 'android_install_apk': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.installApk(args || {});
+        }
+
         case 'web_search': {
             const apiKey = process.env.BRAVE_SEARCH_API_KEY;
             if (!apiKey) return { error: 'BRAVE_SEARCH_API_KEY is not configured' };
@@ -997,6 +1256,7 @@ async function executeTool(toolName, args, context, engine) {
                     cronExpression: args.cron_expression,
                     prompt: args.prompt,
                     enabled: args.enabled !== false,
+                    model: args.model || null,
                     callTo: args.call_to || null,
                     callGreeting: args.call_greeting || null
                 });
@@ -1016,6 +1276,7 @@ async function executeTool(toolName, args, context, engine) {
                     prompt: args.prompt,
                     runAt: args.run_at,
                     oneTime: true,
+                    model: args.model || null,
                     callTo: args.call_to || null,
                     callGreeting: args.call_greeting || null
                 });
@@ -1052,6 +1313,7 @@ async function executeTool(toolName, args, context, engine) {
                 if (args.cron_expression !== undefined) updates.cronExpression = args.cron_expression;
                 if (args.prompt !== undefined) updates.prompt = args.prompt;
                 if (args.enabled !== undefined) updates.enabled = args.enabled;
+                if (args.model !== undefined) updates.model = args.model || null;
                 if (args.call_to !== undefined) updates.callTo = args.call_to || null;
                 if (args.call_greeting !== undefined) updates.callGreeting = args.call_greeting || null;
                 const updated = s.updateTask(args.task_id, userId, updates);