neoagent 2.1.11 → 2.1.13

package/server/services/ai/toolCallSalvage.js

@@ -3,14 +3,38 @@ const { v4: uuidv4 } = require('uuid')
 const INVOKE_OPEN_RE = /(?:[A-Za-z0-9_.-]+:tool_call\s*)?<invoke\s+name="([^"]+)">/g
 const PARAM_OPEN_RE = /<parameter\s+name="([^"]+)">/g
 const PARAM_CLOSED_RE = /<parameter\s+name="([^"]+)">([\s\S]*?)<\/parameter>/g
+const TOOL_WRAPPER_RE = /<\/?[A-Za-z0-9_.-]+:tool_call>/g
+const COMPLETE_INLINE_CALL_RE = /(?:[A-Za-z0-9_.-]+:tool_call\s*)?<invoke\s+name="[^"]+">[\s\S]*?<\/invoke>/g
 const INVOKE_CLOSE = '</invoke>'
 
 function trimLooseControlText(text) {
   return String(text || '')
+    .replace(TOOL_WRAPPER_RE, '')
     .replace(/(?:^|\s)[A-Za-z0-9_.-]+:tool_call\s*$/g, '')
     .trim()
 }
 
+function sanitizeStreamingToolCallText(text) {
+  let visible = String(text || '')
+    .replace(COMPLETE_INLINE_CALL_RE, '')
+    .replace(TOOL_WRAPPER_RE, '')
+
+  const partialStarts = [
+    visible.lastIndexOf('<invoke'),
+    visible.lastIndexOf(':tool_call')
+  ].filter((index) => index >= 0)
+
+  if (partialStarts.length > 0) {
+    const partialStart = Math.max(...partialStarts)
+    const suffix = visible.slice(partialStart)
+    if (!suffix.includes(INVOKE_CLOSE)) {
+      visible = visible.slice(0, partialStart)
+    }
+  }
+
+  return trimLooseControlText(visible).replace(/\n{3,}/g, '\n\n')
+}
+
 function parseParameterMap(body) {
   const args = {}
   let sawClosedParam = false
@@ -113,5 +137,6 @@ function salvageTextToolCalls(content, tools = []) {
 }
 
 module.exports = {
-  salvageTextToolCalls
+  salvageTextToolCalls,
+  sanitizeStreamingToolCallText
 }

package/server/services/ai/toolResult.js

@@ -90,10 +90,12 @@ function compactToolResult(toolName, toolArgs = {}, toolResult, options = {}) {
       break;
 
     case 'android_dump_ui':
+    case 'android_observe':
       envelope = trimObject({
         tool: toolName,
         serial: toolResult?.serial,
         nodeCount: toolResult?.nodeCount,
+        screenshotPath: toolResult?.screenshotPath,
         uiDumpPath: toolResult?.uiDumpPath,
         preview: clampText(JSON.stringify(toolResult?.preview || []).slice(0, Math.floor(softLimit * 0.55)), Math.floor(softLimit * 0.55))
       });

package/server/services/ai/tools.js

@@ -292,6 +292,16 @@ function getAvailableTools(app, options = {}) {
                 }
             }
         },
+        {
+            name: 'android_observe',
+            description: 'Capture the current Android screen end-to-end: fresh screenshot, UI dump path, and a preview of visible UI nodes.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    includeNodes: { type: 'boolean', description: 'Include a preview of parsed UI nodes (default true)' }
+                }
+            }
+        },
         {
             name: 'android_dump_ui',
             description: 'Capture the current Android UIAutomator XML dump and return a preview of the nodes.',
@@ -763,7 +773,7 @@ function getAvailableTools(app, options = {}) {
         },
         {
             name: 'analyze_image',
-            description: 'Analyze an image file using Grok vision. Use this to describe photos, read QR codes, extract text from screenshots, or answer any visual question about an image.',
+            description: 'Analyze an image file using the best available vision-capable model. Use this to describe photos, read QR codes, extract text from screenshots, or answer visual questions.',
             parameters: {
                 type: 'object',
                 properties: {
@@ -940,6 +950,12 @@ async function executeTool(toolName, args, context, engine) {
             return await controller.waitFor(args || {});
         }
 
+        case 'android_observe': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.observe(args || {});
+        }
+
         case 'android_dump_ui': {
             const controller = ac();
             if (!controller) return { error: 'Android controller not available' };
@@ -1541,23 +1557,67 @@ async function executeTool(toolName, args, context, engine) {
         case 'analyze_image': {
             try {
                 if (!fs.existsSync(args.image_path)) return { error: `File not found: ${args.image_path}` };
-                const b64 = fs.readFileSync(args.image_path).toString('base64');
                 const ext = path.extname(args.image_path).toLowerCase();
                 const mimeMap = { '.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg' };
                 const mime = mimeMap[ext] || 'image/jpeg';
+                const question = args.question || 'Describe this image in detail.';
                 const { getProviderForUser } = require('./engine');
-                const { provider: visionProvider, model: visionModel } = await getProviderForUser(userId);
-                const visionResponse = await visionProvider.chat(
-                    [{
-                        role: 'user', content: [
-                            { type: 'text', text: args.question || 'Describe this image in detail.' },
-                            { type: 'image_url', image_url: { url: `data:${mime};base64,${b64}` } }
-                        ]
-                    }],
-                    [],
-                    { model: visionModel }
-                );
-                return { description: visionResponse.content };
+                const { createProviderInstance, getProviderCatalog } = require('./models');
+
+                const attempted = [];
+                const candidates = [];
+
+                try {
+                    const preferred = await getProviderForUser(userId);
+                    candidates.push({
+                        providerName: preferred.providerName,
+                        provider: preferred.provider,
+                    });
+                } catch (err) {
+                    attempted.push(`default-provider lookup failed: ${err.message}`);
+                }
+
+                for (const providerInfo of getProviderCatalog(userId)) {
+                    if (!providerInfo.available) continue;
+                    if (candidates.some((candidate) => candidate.providerName === providerInfo.id)) continue;
+                    if (!['grok', 'openai'].includes(providerInfo.id)) continue;
+                    try {
+                        candidates.push({
+                            providerName: providerInfo.id,
+                            provider: createProviderInstance(providerInfo.id, userId),
+                        });
+                    } catch (err) {
+                        attempted.push(`${providerInfo.id}: ${err.message}`);
+                    }
+                }
+
+                for (const candidate of candidates) {
+                    if (typeof candidate.provider.supportsVision !== 'function' || candidate.provider.supportsVision() !== true) {
+                        attempted.push(`${candidate.providerName}: image analysis is not supported by this provider integration`);
+                        continue;
+                    }
+
+                    try {
+                        const visionResponse = await candidate.provider.analyzeImage({
+                            imagePath: args.image_path,
+                            mimeType: mime,
+                            question,
+                        });
+                        return {
+                            description: visionResponse.content,
+                            model: visionResponse.model || null,
+                            provider: candidate.providerName,
+                        };
+                    } catch (err) {
+                        attempted.push(`${candidate.providerName}: ${err.message}`);
+                    }
+                }
+
+                return {
+                    error: attempted.length > 0
+                        ? `Image analysis failed. ${attempted.join(' | ')}`
+                        : 'No vision-capable provider is currently available. Configure OpenAI or xAI for image analysis.',
+                };
             } catch (err) {
                 return { error: err.message };
             }

package/server/services/android/controller.js

@@ -64,6 +64,25 @@ function commandExists(command) {
   return probe.status === 0;
 }
 
+function parseResolvedLaunchComponent(output, packageName) {
+  const lines = String(output || '')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean);
+  const normalizedPackage = String(packageName || '').trim();
+  const componentPattern = /^[A-Za-z0-9._$]+\/[A-Za-z0-9._$]+$/;
+  const relativePattern = /^[A-Za-z0-9._$]+\/\.[A-Za-z0-9._$]+$/;
+
+  const exact = lines.find((line) =>
+    normalizedPackage
+      ? line.startsWith(`${normalizedPackage}/`)
+      : componentPattern.test(line) || relativePattern.test(line)
+  );
+  if (exact) return exact;
+
+  return lines.find((line) => componentPattern.test(line) || relativePattern.test(line)) || null;
+}
+
 function appendState(patch) {
   const current = readState();
   const next = {
@@ -1085,7 +1104,7 @@ class AndroidController {
   }
 
   async screenshot(options = {}) {
-    const serial = await this.ensureDevice();
+    const serial = options.serial || await this.ensureDevice();
     const filename = `android_${Date.now()}.png`;
     const fullPath = path.join(SCREENSHOTS_DIR, filename);
     await this.#run(`${quoteShell(adbBinary())} -s ${quoteShell(serial)} exec-out screencap -p > ${quoteShell(fullPath)}`, { timeout: 30000 });
@@ -1098,7 +1117,7 @@ class AndroidController {
   }
 
   async dumpUi(options = {}) {
-    const serial = await this.ensureDevice();
+    const serial = options.serial || await this.ensureDevice();
     let xml = await this.#adb(serial, 'shell uiautomator dump --compressed /dev/tty', { timeout: 30000 });
     if (!String(xml || '').includes('<hierarchy')) {
       const remote = '/sdcard/neoagent-ui.xml';
@@ -1121,6 +1140,63 @@ class AndroidController {
     };
   }
 
+  async #captureObservation(serial, options = {}) {
+    const resolvedSerial = serial || await this.ensureDevice();
+    const observation = {
+      serial: resolvedSerial,
+      screenshotPath: null,
+      fullPath: null,
+      uiDumpPath: null,
+      nodeCount: null,
+      preview: undefined,
+      observationWarnings: [],
+    };
+
+    if (options.screenshot !== false) {
+      try {
+        const shot = await this.screenshot({ serial: resolvedSerial });
+        observation.screenshotPath = shot?.screenshotPath || null;
+        observation.fullPath = shot?.fullPath || null;
+      } catch (err) {
+        observation.observationWarnings.push(`screenshot: ${err.message}`);
+      }
+    }
+
+    try {
+      const dump = await this.dumpUi({
+        serial: resolvedSerial,
+        includeNodes: options.includeNodes !== false,
+      });
+      observation.uiDumpPath = dump.uiDumpPath;
+      observation.nodeCount = dump.nodeCount;
+      observation.preview = dump.preview;
+    } catch (err) {
+      observation.observationWarnings.push(`ui_dump: ${err.message}`);
+    }
+
+    if (observation.observationWarnings.length === 0) {
+      delete observation.observationWarnings;
+    }
+
+    return observation;
+  }
+
+  async observe(options = {}) {
+    const serial = options.serial || await this.ensureDevice();
+    const observation = await this.#captureObservation(serial, options);
+    if (!observation.screenshotPath && !observation.uiDumpPath) {
+      throw new Error(
+        Array.isArray(observation.observationWarnings) && observation.observationWarnings.length > 0
+          ? observation.observationWarnings.join(' | ')
+          : 'Unable to capture Android observation',
+      );
+    }
+    return {
+      success: true,
+      ...observation,
+    };
+  }
+
   async #resolveSelector(args = {}) {
     const dump = await this.dumpUi({ includeNodes: false });
     const selector = {
@@ -1145,27 +1221,27 @@ class AndroidController {
     let y = Number(args.y);
     let node = null;
     let serial = await this.ensureDevice();
-    let uiDumpPath = null;
+    let resolvedFromUiDumpPath = null;
 
     if (!Number.isFinite(x) || !Number.isFinite(y)) {
       const resolved = await this.#resolveSelector(args);
       serial = resolved.serial;
       node = resolved.node;
-      uiDumpPath = resolved.uiDumpPath;
+      resolvedFromUiDumpPath = resolved.uiDumpPath;
       x = node.bounds.centerX;
       y = node.bounds.centerY;
     }
 
     await this.#adb(serial, `shell input tap ${Math.round(x)} ${Math.round(y)}`, { timeout: 15000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
       x: Math.round(x),
       y: Math.round(y),
       target: summarizeNode(node),
-      uiDumpPath,
-      screenshotPath: shot.screenshotPath,
+      resolvedFromUiDumpPath,
+      ...observation,
     };
   }
 
@@ -1174,13 +1250,13 @@ class AndroidController {
     let y = Number(args.y);
     let node = null;
     let serial = await this.ensureDevice();
-    let uiDumpPath = null;
+    let resolvedFromUiDumpPath = null;
 
     if (!Number.isFinite(x) || !Number.isFinite(y)) {
       const resolved = await this.#resolveSelector(args);
       serial = resolved.serial;
       node = resolved.node;
-      uiDumpPath = resolved.uiDumpPath;
+      resolvedFromUiDumpPath = resolved.uiDumpPath;
       x = node.bounds.centerX;
       y = node.bounds.centerY;
     }
@@ -1191,7 +1267,7 @@ class AndroidController {
       `shell input swipe ${Math.round(x)} ${Math.round(y)} ${Math.round(x)} ${Math.round(y)} ${Math.round(durationMs)}`,
       { timeout: Math.max(15000, durationMs + 5000) },
     );
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
@@ -1199,8 +1275,8 @@ class AndroidController {
       y: Math.round(y),
       durationMs,
       target: summarizeNode(node),
-      uiDumpPath,
-      screenshotPath: shot.screenshotPath,
+      resolvedFromUiDumpPath,
+      ...observation,
     };
   }
 
@@ -1225,12 +1301,12 @@ class AndroidController {
     if (args.pressEnter) {
       await this.#adb(serial, 'shell input keyevent 66', { timeout: 10000 });
     }
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
       typed: args.text || '',
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1245,11 +1321,11 @@ class AndroidController {
       throw new Error('x1, y1, x2, and y2 are required for android_swipe');
     }
     await this.#adb(serial, `shell input swipe ${Math.round(x1)} ${Math.round(y1)} ${Math.round(x2)} ${Math.round(y2)} ${Math.round(duration)}`, { timeout: 15000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1259,13 +1335,13 @@ class AndroidController {
     const keyCode = Number.isFinite(Number(raw)) ? Number(raw) : (DEFAULT_KEYEVENTS[raw] || null);
     if (!keyCode) throw new Error(`Unsupported Android key: ${args.key}`);
     await this.#adb(serial, `shell input keyevent ${keyCode}`, { timeout: 10000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
       key: args.key,
       keyCode,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1285,13 +1361,15 @@ class AndroidController {
         clickable: args.clickable,
       });
       if (node) {
-        const shot = args.screenshot === false ? null : await this.screenshot();
+        const observation = await this.#captureObservation(dump.serial, {
+          screenshot: args.screenshot !== false,
+        });
         return {
           success: true,
           serial: dump.serial,
           matched: summarizeNode(node),
-          uiDumpPath: dump.uiDumpPath,
-          screenshotPath: shot?.screenshotPath || null,
+          matchedFromUiDumpPath: dump.uiDumpPath,
+          ...observation,
         };
       }
       await sleep(intervalMs);
@@ -1305,17 +1383,30 @@ class AndroidController {
     if (args.activity) {
       await this.#adb(serial, `shell am start -n ${quoteShell(`${args.packageName}/${args.activity}`)}`, { timeout: 20000 });
     } else if (args.packageName) {
-      await this.#adb(serial, `shell monkey -p ${quoteShell(args.packageName)} -c android.intent.category.LAUNCHER 1`, { timeout: 30000 });
+      const resolved = await this.#runAllowFailure(
+        `${quoteShell(adbBinary())} -s ${quoteShell(serial)} shell cmd package resolve-activity --brief -c android.intent.category.LAUNCHER ${quoteShell(args.packageName)}`,
+        { timeout: 15000 },
+      );
+      const component = parseResolvedLaunchComponent(
+        `${resolved.stdout || ''}\n${resolved.stderr || ''}`,
+        args.packageName,
+      );
+
+      if (component) {
+        await this.#adb(serial, `shell am start -n ${quoteShell(component)}`, { timeout: 20000 });
+      } else {
+        await this.#adb(serial, `shell monkey -p ${quoteShell(args.packageName)} -c android.intent.category.LAUNCHER 1`, { timeout: 30000 });
+      }
     } else {
       throw new Error('packageName is required for android_open_app');
     }
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
       packageName: args.packageName,
       activity: args.activity || null,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1335,11 +1426,11 @@ class AndroidController {
     }
 
     await this.#adb(serial, parts.join(' '), { timeout: 20000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1380,13 +1471,19 @@ class AndroidController {
 
     const timeout = Math.max(1000, Number(args.timeoutMs) || 20000);
     const stdout = await this.#adb(serial, `shell ${quoteShell(command)}`, { timeout });
-    const shot = args.screenshot === true ? await this.screenshot() : null;
+    const observation = args.screenshot === true
+      ? await this.#captureObservation(serial)
+      : null;
     return {
       success: true,
       serial,
       command,
       stdout,
-      screenshotPath: shot?.screenshotPath || null,
+      screenshotPath: observation?.screenshotPath || null,
+      fullPath: observation?.fullPath || null,
+      uiDumpPath: observation?.uiDumpPath || null,
+      nodeCount: observation?.nodeCount,
+      preview: observation?.preview,
     };
   }
 
@@ -1458,6 +1555,7 @@ module.exports = {
   configuredSystemImagePackage,
   configuredSystemImagePlatform,
   formatSystemImageError,
+  parseResolvedLaunchComponent,
   parseLatestCmdlineToolsUrl,
   parseSystemImages,
   sanitizeUiXml,

package/server/services/websocket.js

@@ -76,6 +76,25 @@ function setupWebSocket(io, services) {
           }
         }
 
+        const activeRun = agentEngine.findSteerableRunForUser(userId, 'web');
+        if (activeRun) {
+          const queued = agentEngine.enqueueSteering(activeRun.runId, task, {
+            platform: 'web',
+            socketId: socket.id
+          });
+          if (queued) {
+            db.prepare('INSERT INTO conversation_history (user_id, agent_run_id, role, content, metadata) VALUES (?, ?, ?, ?, ?)')
+              .run(
+                userId,
+                activeRun.runId,
+                'user',
+                task,
+                JSON.stringify({ platform: 'web', steering: true })
+              );
+            return;
+          }
+        }
+
         db.prepare('INSERT INTO conversation_history (user_id, role, content, metadata) VALUES (?, ?, ?, ?)')
           .run(userId, 'user', task, JSON.stringify({ platform: 'web' }));