neoagent 2.1.12 → 2.1.13

package/server/routes/android.js

@@ -85,6 +85,15 @@ router.post('/screenshot', async (req, res) => {
   }
 });
 
+router.post('/observe', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.observe(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
 router.post('/ui-dump', async (req, res) => {
   try {
     const controller = req.app.locals.androidController;

package/server/services/ai/engine.js

@@ -163,6 +163,86 @@ class AgentEngine {
     return this.activeRuns.get(runId) || null;
   }
 
+  findActiveRunForUser(userId, predicate = null) {
+    let candidate = null;
+    for (const [runId, runMeta] of this.activeRuns.entries()) {
+      if (runMeta.userId !== userId || runMeta.aborted) continue;
+      if (typeof predicate === 'function' && !predicate(runMeta, runId)) continue;
+      if (!candidate || (runMeta.startedAt || 0) >= (candidate.startedAt || 0)) {
+        candidate = { runId, ...runMeta };
+      }
+    }
+    return candidate;
+  }
+
+  findSteerableRunForUser(userId, triggerSource = 'web') {
+    return this.findActiveRunForUser(
+      userId,
+      (runMeta) => runMeta.triggerSource === triggerSource && runMeta.triggerType === 'user'
+    );
+  }
+
+  enqueueSteering(runId, content, metadata = {}) {
+    const runMeta = this.getRunMeta(runId);
+    const trimmed = typeof content === 'string' ? content.trim() : '';
+    if (!runMeta || runMeta.aborted || !trimmed) return null;
+
+    const item = {
+      id: uuidv4(),
+      content: trimmed,
+      metadata,
+      createdAt: new Date().toISOString()
+    };
+
+    runMeta.steeringQueue.push(item);
+    this.emit(runMeta.userId, 'run:steer_queued', {
+      runId,
+      content: item.content,
+      pendingCount: runMeta.steeringQueue.length
+    });
+
+    return {
+      runId,
+      pendingCount: runMeta.steeringQueue.length,
+      item
+    };
+  }
+
+  applyQueuedSteering(runId, messages, { userId, conversationId }) {
+    const runMeta = this.getRunMeta(runId);
+    if (!runMeta?.steeringQueue?.length) {
+      return { messages, appliedCount: 0 };
+    }
+
+    const queued = runMeta.steeringQueue.splice(0, runMeta.steeringQueue.length);
+    messages.push({
+      role: 'system',
+      content: [
+        'The user sent follow-up messages while you were already working.',
+        'Treat them as steering or next-up context for the same conversation.',
+        'If a message materially changes the active task, incorporate it now.',
+        'If it is unrelated or better handled after the current task, finish the current work first and then address it.'
+      ].join(' ')
+    });
+
+    for (const entry of queued) {
+      messages.push({ role: 'user', content: entry.content });
+      if (conversationId) {
+        db.prepare('INSERT INTO conversation_messages (conversation_id, role, content) VALUES (?, ?, ?)')
+          .run(conversationId, 'user', entry.content);
+      }
+    }
+
+    this.emit(userId, 'run:steer_applied', {
+      runId,
+      count: queued.length,
+      pendingCount: runMeta.steeringQueue.length,
+      latestContent: queued[queued.length - 1]?.content || ''
+    });
+
+    return { messages, appliedCount: queued.length };
+  }
+
   isRunStopped(runId) {
     return this.getRunMeta(runId)?.aborted === true;
   }
@@ -321,8 +401,12 @@ class AgentEngine {
       status: 'running',
       aborted: false,
       messagingSent: false,
+      triggerType,
+      triggerSource,
+      startedAt: Date.now(),
       lastToolName: null,
       lastToolTarget: null,
+      steeringQueue: [],
       toolPids: new Set()
     });
     this.emit(userId, 'run:start', { runId, title: runTitle, model, triggerType, triggerSource });
@@ -372,6 +456,12 @@ class AgentEngine {
         if (this.isRunStopped(runId)) break;
         iteration++;
 
+        const steeringAtLoopStart = this.applyQueuedSteering(runId, messages, {
+          userId,
+          conversationId
+        });
+        messages = steeringAtLoopStart.messages;
+
         let metrics = this.estimatePromptMetrics(messages, tools);
         const contextWindow = provider.getContextWindow(model);
         if (metrics.totalEstimatedTokens > contextWindow * 0.7) {
@@ -513,7 +603,19 @@ class AgentEngine {
             );
         }
 
-        if (!response.toolCalls || response.toolCalls.length === 0) break;
+        if (!response.toolCalls || response.toolCalls.length === 0) {
+          const steeringAfterResponse = this.applyQueuedSteering(runId, messages, {
+            userId,
+            conversationId
+          });
+          messages = steeringAfterResponse.messages;
+          if (steeringAfterResponse.appliedCount > 0) {
+            iteration = Math.max(0, iteration - 1);
+            lastContent = '';
+            continue;
+          }
+          break;
+        }
 
         for (const toolCall of response.toolCalls) {
           if (this.isRunStopped(runId)) break;

package/server/services/ai/providers/base.js

@@ -1,4 +1,9 @@
 class BaseProvider {
+  static readImageAsBase64(imagePath) {
+    const fs = require('fs');
+    return fs.readFileSync(imagePath).toString('base64');
+  }
+
   constructor(config = {}) {
     this.config = config;
     this.name = 'base';
@@ -36,6 +41,18 @@ class BaseProvider {
   getContextWindow(model) {
     return 128000;
   }
+
+  supportsVision() {
+    return false;
+  }
+
+  getDefaultVisionModel() {
+    return null;
+  }
+
+  async analyzeImage(_options = {}) {
+    throw new Error(`Provider '${this.name}' does not support image analysis`);
+  }
 }
 
 module.exports = { BaseProvider };

package/server/services/ai/providers/grok.js

@@ -15,6 +15,14 @@ class GrokProvider extends BaseProvider {
     return 131072; // grok-4 context window
   }
 
+  supportsVision() {
+    return true;
+  }
+
+  getDefaultVisionModel() {
+    return 'grok-4.20-beta-latest-non-reasoning';
+  }
+
   _buildParams(model, messages, tools, options) {
     const params = {
       model,
@@ -116,6 +124,32 @@ class GrokProvider extends BaseProvider {
       }
     }));
   }
+
+  async analyzeImage(options = {}) {
+    const model = options.model || this.getDefaultVisionModel();
+    const b64 = BaseProvider.readImageAsBase64(options.imagePath);
+    const response = await this.client.chat.completions.create({
+      model,
+      max_tokens: options.maxTokens || 4096,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: options.question || 'Describe this image in detail.' },
+          {
+            type: 'image_url',
+            image_url: {
+              url: `data:${options.mimeType || 'image/jpeg'};base64,${b64}`
+            }
+          }
+        ]
+      }]
+    });
+
+    return {
+      content: response.choices[0]?.message?.content || '',
+      model: response.model || model,
+    };
+  }
 }
 
 module.exports = { GrokProvider };

package/server/services/ai/providers/openai.js

@@ -48,6 +48,14 @@ class OpenAIProvider extends BaseProvider {
     return 128000;
   }
 
+  supportsVision() {
+    return true;
+  }
+
+  getDefaultVisionModel() {
+    return 'gpt-4.1-mini';
+  }
+
   _buildParams(model, messages, tools, options) {
     const isReasoning = this.isReasoningModel(model);
     // Reasoning models (GPT-5, o-series): use developer role for system messages
@@ -163,6 +171,32 @@ class OpenAIProvider extends BaseProvider {
       }
     }
   }
+
+  async analyzeImage(options = {}) {
+    const model = options.model || this.getDefaultVisionModel();
+    const b64 = BaseProvider.readImageAsBase64(options.imagePath);
+    const response = await this.client.chat.completions.create({
+      model,
+      max_tokens: options.maxTokens || 4096,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: options.question || 'Describe this image in detail.' },
+          {
+            type: 'image_url',
+            image_url: {
+              url: `data:${options.mimeType || 'image/jpeg'};base64,${b64}`
+            }
+          }
+        ]
+      }]
+    });
+
+    return {
+      content: response.choices[0]?.message?.content || '',
+      model: response.model || model,
+    };
+  }
 }
 
 module.exports = { OpenAIProvider };

package/server/services/ai/systemPrompt.js

@@ -41,6 +41,7 @@ When prior context makes the goal clear, act on it. Only ask a clarifying questi
 
 REPORT ACTUAL RESULTS
 When a tool returns data, share the relevant parts — summarized if large, direct if short. Never paste raw JSON as the answer. Never narrate what you're about to do at length before doing it.
+Never promise an action in the final answer unless you already took that action in this run. Do not say "I'll check", "I'll fix it", or "I'll send it" and then stop. Either do it first or say you have not done it yet.
 
 DON'T REPEAT YOURSELF
 State a limitation or error once. If the user pushes back, try a different approach before restating the same failure. Repeating the same dead-end across five messages is useless.

package/server/services/ai/toolResult.js

@@ -90,10 +90,12 @@ function compactToolResult(toolName, toolArgs = {}, toolResult, options = {}) {
       break;
 
     case 'android_dump_ui':
+    case 'android_observe':
       envelope = trimObject({
         tool: toolName,
         serial: toolResult?.serial,
         nodeCount: toolResult?.nodeCount,
+        screenshotPath: toolResult?.screenshotPath,
         uiDumpPath: toolResult?.uiDumpPath,
         preview: clampText(JSON.stringify(toolResult?.preview || []).slice(0, Math.floor(softLimit * 0.55)), Math.floor(softLimit * 0.55))
       });

package/server/services/ai/tools.js

@@ -292,6 +292,16 @@ function getAvailableTools(app, options = {}) {
                 }
             }
         },
+        {
+            name: 'android_observe',
+            description: 'Capture the current Android screen end-to-end: fresh screenshot, UI dump path, and a preview of visible UI nodes.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    includeNodes: { type: 'boolean', description: 'Include a preview of parsed UI nodes (default true)' }
+                }
+            }
+        },
         {
             name: 'android_dump_ui',
             description: 'Capture the current Android UIAutomator XML dump and return a preview of the nodes.',
@@ -763,7 +773,7 @@ function getAvailableTools(app, options = {}) {
         },
         {
             name: 'analyze_image',
-            description: 'Analyze an image file using Grok vision. Use this to describe photos, read QR codes, extract text from screenshots, or answer any visual question about an image.',
+            description: 'Analyze an image file using the best available vision-capable model. Use this to describe photos, read QR codes, extract text from screenshots, or answer visual questions.',
             parameters: {
                 type: 'object',
                 properties: {
@@ -940,6 +950,12 @@ async function executeTool(toolName, args, context, engine) {
             return await controller.waitFor(args || {});
         }
 
+        case 'android_observe': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.observe(args || {});
+        }
+
         case 'android_dump_ui': {
             const controller = ac();
             if (!controller) return { error: 'Android controller not available' };
@@ -1541,23 +1557,67 @@ async function executeTool(toolName, args, context, engine) {
         case 'analyze_image': {
             try {
                 if (!fs.existsSync(args.image_path)) return { error: `File not found: ${args.image_path}` };
-                const b64 = fs.readFileSync(args.image_path).toString('base64');
                 const ext = path.extname(args.image_path).toLowerCase();
                 const mimeMap = { '.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg' };
                 const mime = mimeMap[ext] || 'image/jpeg';
+                const question = args.question || 'Describe this image in detail.';
                 const { getProviderForUser } = require('./engine');
-                const { provider: visionProvider, model: visionModel } = await getProviderForUser(userId);
-                const visionResponse = await visionProvider.chat(
-                    [{
-                        role: 'user', content: [
-                            { type: 'text', text: args.question || 'Describe this image in detail.' },
-                            { type: 'image_url', image_url: { url: `data:${mime};base64,${b64}` } }
-                        ]
-                    }],
-                    [],
-                    { model: visionModel }
-                );
-                return { description: visionResponse.content };
+                const { createProviderInstance, getProviderCatalog } = require('./models');
+
+                const attempted = [];
+                const candidates = [];
+
+                try {
+                    const preferred = await getProviderForUser(userId);
+                    candidates.push({
+                        providerName: preferred.providerName,
+                        provider: preferred.provider,
+                    });
+                } catch (err) {
+                    attempted.push(`default-provider lookup failed: ${err.message}`);
+                }
+
+                for (const providerInfo of getProviderCatalog(userId)) {
+                    if (!providerInfo.available) continue;
+                    if (candidates.some((candidate) => candidate.providerName === providerInfo.id)) continue;
+                    if (!['grok', 'openai'].includes(providerInfo.id)) continue;
+                    try {
+                        candidates.push({
+                            providerName: providerInfo.id,
+                            provider: createProviderInstance(providerInfo.id, userId),
+                        });
+                    } catch (err) {
+                        attempted.push(`${providerInfo.id}: ${err.message}`);
+                    }
+                }
+
+                for (const candidate of candidates) {
+                    if (typeof candidate.provider.supportsVision !== 'function' || candidate.provider.supportsVision() !== true) {
+                        attempted.push(`${candidate.providerName}: image analysis is not supported by this provider integration`);
+                        continue;
+                    }
+
+                    try {
+                        const visionResponse = await candidate.provider.analyzeImage({
+                            imagePath: args.image_path,
+                            mimeType: mime,
+                            question,
+                        });
+                        return {
+                            description: visionResponse.content,
+                            model: visionResponse.model || null,
+                            provider: candidate.providerName,
+                        };
+                    } catch (err) {
+                        attempted.push(`${candidate.providerName}: ${err.message}`);
+                    }
+                }
+
+                return {
+                    error: attempted.length > 0
+                        ? `Image analysis failed. ${attempted.join(' | ')}`
+                        : 'No vision-capable provider is currently available. Configure OpenAI or xAI for image analysis.',
+                };
             } catch (err) {
                 return { error: err.message };
             }

package/server/services/android/controller.js

@@ -1104,7 +1104,7 @@ class AndroidController {
   }
 
   async screenshot(options = {}) {
-    const serial = await this.ensureDevice();
+    const serial = options.serial || await this.ensureDevice();
     const filename = `android_${Date.now()}.png`;
     const fullPath = path.join(SCREENSHOTS_DIR, filename);
     await this.#run(`${quoteShell(adbBinary())} -s ${quoteShell(serial)} exec-out screencap -p > ${quoteShell(fullPath)}`, { timeout: 30000 });
@@ -1117,7 +1117,7 @@ class AndroidController {
   }
 
   async dumpUi(options = {}) {
-    const serial = await this.ensureDevice();
+    const serial = options.serial || await this.ensureDevice();
     let xml = await this.#adb(serial, 'shell uiautomator dump --compressed /dev/tty', { timeout: 30000 });
     if (!String(xml || '').includes('<hierarchy')) {
       const remote = '/sdcard/neoagent-ui.xml';
@@ -1140,6 +1140,63 @@ class AndroidController {
     };
   }
 
+  async #captureObservation(serial, options = {}) {
+    const resolvedSerial = serial || await this.ensureDevice();
+    const observation = {
+      serial: resolvedSerial,
+      screenshotPath: null,
+      fullPath: null,
+      uiDumpPath: null,
+      nodeCount: null,
+      preview: undefined,
+      observationWarnings: [],
+    };
+
+    if (options.screenshot !== false) {
+      try {
+        const shot = await this.screenshot({ serial: resolvedSerial });
+        observation.screenshotPath = shot?.screenshotPath || null;
+        observation.fullPath = shot?.fullPath || null;
+      } catch (err) {
+        observation.observationWarnings.push(`screenshot: ${err.message}`);
+      }
+    }
+
+    try {
+      const dump = await this.dumpUi({
+        serial: resolvedSerial,
+        includeNodes: options.includeNodes !== false,
+      });
+      observation.uiDumpPath = dump.uiDumpPath;
+      observation.nodeCount = dump.nodeCount;
+      observation.preview = dump.preview;
+    } catch (err) {
+      observation.observationWarnings.push(`ui_dump: ${err.message}`);
+    }
+
+    if (observation.observationWarnings.length === 0) {
+      delete observation.observationWarnings;
+    }
+
+    return observation;
+  }
+
+  async observe(options = {}) {
+    const serial = options.serial || await this.ensureDevice();
+    const observation = await this.#captureObservation(serial, options);
+    if (!observation.screenshotPath && !observation.uiDumpPath) {
+      throw new Error(
+        Array.isArray(observation.observationWarnings) && observation.observationWarnings.length > 0
+          ? observation.observationWarnings.join(' | ')
+          : 'Unable to capture Android observation',
+      );
+    }
+    return {
+      success: true,
+      ...observation,
+    };
+  }
+
   async #resolveSelector(args = {}) {
     const dump = await this.dumpUi({ includeNodes: false });
     const selector = {
@@ -1164,27 +1221,27 @@ class AndroidController {
     let y = Number(args.y);
     let node = null;
     let serial = await this.ensureDevice();
-    let uiDumpPath = null;
+    let resolvedFromUiDumpPath = null;
 
     if (!Number.isFinite(x) || !Number.isFinite(y)) {
       const resolved = await this.#resolveSelector(args);
       serial = resolved.serial;
       node = resolved.node;
-      uiDumpPath = resolved.uiDumpPath;
+      resolvedFromUiDumpPath = resolved.uiDumpPath;
       x = node.bounds.centerX;
       y = node.bounds.centerY;
     }
 
     await this.#adb(serial, `shell input tap ${Math.round(x)} ${Math.round(y)}`, { timeout: 15000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
       x: Math.round(x),
       y: Math.round(y),
       target: summarizeNode(node),
-      uiDumpPath,
-      screenshotPath: shot.screenshotPath,
+      resolvedFromUiDumpPath,
+      ...observation,
     };
   }
 
@@ -1193,13 +1250,13 @@ class AndroidController {
     let y = Number(args.y);
     let node = null;
     let serial = await this.ensureDevice();
-    let uiDumpPath = null;
+    let resolvedFromUiDumpPath = null;
 
     if (!Number.isFinite(x) || !Number.isFinite(y)) {
       const resolved = await this.#resolveSelector(args);
       serial = resolved.serial;
       node = resolved.node;
-      uiDumpPath = resolved.uiDumpPath;
+      resolvedFromUiDumpPath = resolved.uiDumpPath;
       x = node.bounds.centerX;
       y = node.bounds.centerY;
     }
@@ -1210,7 +1267,7 @@ class AndroidController {
       `shell input swipe ${Math.round(x)} ${Math.round(y)} ${Math.round(x)} ${Math.round(y)} ${Math.round(durationMs)}`,
       { timeout: Math.max(15000, durationMs + 5000) },
     );
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
@@ -1218,8 +1275,8 @@ class AndroidController {
       y: Math.round(y),
       durationMs,
       target: summarizeNode(node),
-      uiDumpPath,
-      screenshotPath: shot.screenshotPath,
+      resolvedFromUiDumpPath,
+      ...observation,
     };
   }
 
@@ -1244,12 +1301,12 @@ class AndroidController {
     if (args.pressEnter) {
       await this.#adb(serial, 'shell input keyevent 66', { timeout: 10000 });
     }
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
       typed: args.text || '',
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1264,11 +1321,11 @@ class AndroidController {
       throw new Error('x1, y1, x2, and y2 are required for android_swipe');
     }
     await this.#adb(serial, `shell input swipe ${Math.round(x1)} ${Math.round(y1)} ${Math.round(x2)} ${Math.round(y2)} ${Math.round(duration)}`, { timeout: 15000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1278,13 +1335,13 @@ class AndroidController {
     const keyCode = Number.isFinite(Number(raw)) ? Number(raw) : (DEFAULT_KEYEVENTS[raw] || null);
     if (!keyCode) throw new Error(`Unsupported Android key: ${args.key}`);
     await this.#adb(serial, `shell input keyevent ${keyCode}`, { timeout: 10000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
       key: args.key,
       keyCode,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1304,13 +1361,15 @@ class AndroidController {
         clickable: args.clickable,
       });
       if (node) {
-        const shot = args.screenshot === false ? null : await this.screenshot();
+        const observation = await this.#captureObservation(dump.serial, {
+          screenshot: args.screenshot !== false,
+        });
         return {
           success: true,
           serial: dump.serial,
           matched: summarizeNode(node),
-          uiDumpPath: dump.uiDumpPath,
-          screenshotPath: shot?.screenshotPath || null,
+          matchedFromUiDumpPath: dump.uiDumpPath,
+          ...observation,
         };
       }
       await sleep(intervalMs);
@@ -1341,13 +1400,13 @@ class AndroidController {
     } else {
       throw new Error('packageName is required for android_open_app');
     }
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
       packageName: args.packageName,
       activity: args.activity || null,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1367,11 +1426,11 @@ class AndroidController {
     }
 
     await this.#adb(serial, parts.join(' '), { timeout: 20000 });
-    const shot = await this.screenshot();
+    const observation = await this.#captureObservation(serial);
     return {
       success: true,
       serial,
-      screenshotPath: shot.screenshotPath,
+      ...observation,
     };
   }
 
@@ -1412,13 +1471,19 @@ class AndroidController {
 
     const timeout = Math.max(1000, Number(args.timeoutMs) || 20000);
     const stdout = await this.#adb(serial, `shell ${quoteShell(command)}`, { timeout });
-    const shot = args.screenshot === true ? await this.screenshot() : null;
+    const observation = args.screenshot === true
+      ? await this.#captureObservation(serial)
+      : null;
     return {
       success: true,
       serial,
       command,
       stdout,
-      screenshotPath: shot?.screenshotPath || null,
+      screenshotPath: observation?.screenshotPath || null,
+      fullPath: observation?.fullPath || null,
+      uiDumpPath: observation?.uiDumpPath || null,
+      nodeCount: observation?.nodeCount,
+      preview: observation?.preview,
     };
   }